From 917c00c23a7f68c3fdda9d06b4583e35db29c68a Mon Sep 17 00:00:00 2001 From: Mason Payne Date: Fri, 8 Mar 2024 01:13:27 -0700 Subject: [PATCH] make the chat work --- LLMMapper/llmMapper.go | 134 +++++++++++++++++++++++++++++++++ cmd/chat/chat.go | 114 +++++++++++++++++++++++++++- main.go | 91 +--------------------- promptBuilder/promptBuilder.go | 21 ++++++ transcript.md | 128 +++++++++++++++++++++++++++++++ 5 files changed, 398 insertions(+), 90 deletions(-) create mode 100644 LLMMapper/llmMapper.go create mode 100644 promptBuilder/promptBuilder.go create mode 100644 transcript.md diff --git a/LLMMapper/llmMapper.go b/LLMMapper/llmMapper.go new file mode 100644 index 0000000..95e8beb --- /dev/null +++ b/LLMMapper/llmMapper.go @@ -0,0 +1,134 @@ +package LLMMapper + +import ( + "context" + "encoding/json" + "fmt" + "github.com/pkoukk/tiktoken-go" + "github.com/sashabaranov/go-openai" + "os" + "strings" + "time" +) + +const encodingName = "gpt-4" +const model = openai.GPT4TurboPreview +const maxTokens = 4096 + +// const maxTokens = 128000 +const temperature = 0.3 + +func GetTokenCount(input string) (int, error) { + tke, err := tiktoken.EncodingForModel(encodingName) // cached in "TIKTOKEN_CACHE_DIR" + if err != nil { + return 0, fmt.Errorf("error getting encoding: %w", err) + } + token := tke.Encode(input, nil, nil) + return len(token), nil +} + +// SinglePromptInteraction calls openai chat endpoint with just a system prompt and a user prompt and returns the response +func SinglePromptInteraction(systemPrompt, prompt string) (openai.ChatCompletionResponse, error) { + return singlePromptInteraction(systemPrompt, prompt, 5) +} + +// singlePromptInteraction calls openai chat endpoint with just a system prompt and a user prompt and returns the response +// it also attempts 5 retries spaced 5 seconds apart in the case of rate limiting errors +func singlePromptInteraction(systemPrompt, prompt string, retries int) (openai.ChatCompletionResponse, error) { + + client := openai.NewClient(os.Getenv("OPENAI_API_KEY")) + + messages := []openai.ChatCompletionMessage{ + { + Role: openai.ChatMessageRoleSystem, + Content: systemPrompt, + }, + { + Role: openai.ChatMessageRoleUser, + Content: prompt, + }, + } + + previousTokenCount, err := GetPreviousTokenUsage(messages) + if err != nil { + return openai.ChatCompletionResponse{}, fmt.Errorf("error getting previous token usage: %w", err) + } + + messageTokenSize := maxTokens - previousTokenCount + + resp, err := client.CreateChatCompletion( + context.Background(), + openai.ChatCompletionRequest{ + Model: model, // switch to the configured Model + Messages: messages, + MaxTokens: messageTokenSize, // might want to think about how to reduce this + Temperature: temperature, + }, + ) + if err != nil { + // if 429, wait and try again + if strings.Contains(err.Error(), "429") && retries > 0 { + seconds := (1 / retries) * 60 // back off for each retry e.g. 12, 15, 20, 30, 60 + fmt.Printf("429 error, waiting %v seconds...\n", seconds) + time.Sleep(time.Duration(seconds) * time.Second) + return singlePromptInteraction(systemPrompt, prompt, retries-1) // TODO: establish base case to prevent forever retrying + } + return openai.ChatCompletionResponse{}, fmt.Errorf("ChatCompletion request error: %w", err) + } + + return resp, nil +} + +func GetPreviousTokenUsage(messages []openai.ChatCompletionMessage) (int, error) { + + messagesRaw, err := json.Marshal(messages) + if err != nil { + return 0, fmt.Errorf("error marshalling messages: %w", err) + } + + length, err := GetTokenCount(string(messagesRaw)) + if err != nil { + return 0, fmt.Errorf("error getting token count: %w", err) + } + //fmt.Printf("Token Count: %v\n", length) + return length, nil +} + +// SendPrompt calls openai chat endpoint with a list of messages and returns the response +func SendPrompt(messages []openai.ChatCompletionMessage) (openai.ChatCompletionResponse, error) { + return sendPrompt(messages, 5) +} + +func sendPrompt(messages []openai.ChatCompletionMessage, retries int) (openai.ChatCompletionResponse, error) { + client := openai.NewClient(os.Getenv("OPENAI_API_KEY")) + + previousTokenCount, err := GetPreviousTokenUsage(messages) + if err != nil { + return openai.ChatCompletionResponse{}, fmt.Errorf("error getting previous token usage: %w", err) + } + + messageTokenSize := maxTokens - previousTokenCount + fmt.Println("messageTokenSize: ", messageTokenSize) + + resp, err := client.CreateChatCompletion( + context.Background(), + openai.ChatCompletionRequest{ + Model: model, // switch to the configured Model + Messages: messages, + MaxTokens: messageTokenSize, + Temperature: temperature, + }, + ) + if err != nil { + // if 429, wait and try again + if strings.Contains(err.Error(), "429") && retries > 0 { + seconds := (1 / retries) * 60 // back off for each retry e.g. 12, 15, 20, 30, 60 + fmt.Printf("429 error, waiting %v seconds...\n", seconds) + time.Sleep(time.Duration(seconds) * time.Second) + return sendPrompt(messages, retries-1) // TODO: establish base case to prevent forever retrying + } + return openai.ChatCompletionResponse{}, fmt.Errorf("ChatCompletion request error: %w", err) + } + + return resp, nil +} diff --git a/cmd/chat/chat.go b/cmd/chat/chat.go index 5b9e0e4..5d3e31b 100644 --- a/cmd/chat/chat.go +++ b/cmd/chat/chat.go @@ -2,7 +2,11 @@ package main import ( "bufio" + "ctxGPT/LLMMapper" + "ctxGPT/promptBuilder" + "encoding/json" "fmt" + "github.com/sashabaranov/go-openai" "os" "strings" ) @@ -12,18 +16,122 @@ func main() { fmt.Println("Go CLI Chat App") fmt.Println("---------------------") + systemPrompt := "" + + messages := []openai.ChatCompletionMessage{ + { + Role: openai.ChatMessageRoleSystem, + Content: systemPrompt, + }, + } + for { fmt.Print("You: ") text, _ := reader.ReadString('\n') text = strings.TrimSpace(text) // Remove leading and trailing whitespace - // Process the input. For now, we'll just echo it back. - fmt.Printf("App: You said, %s\n", text) - // Check if the user wants to exit. if text == "exit" { fmt.Println("Exiting chat...") break } + + messages = append(messages, openai.ChatCompletionMessage{ + Role: openai.ChatMessageRoleUser, + Content: text, + }) + + resp, err := LLMMapper.SendPrompt(messages) + if err != nil { + fmt.Println(err) + } + + if len(resp.Choices) == 0 { + fmt.Println("No choices returned") + continue + } + + fmt.Println("AI: ", resp.Choices[0].Message.Content) + //fmt.Println("Finish Reason: ", resp.Choices[0].FinishReason) + messages = append(messages, resp.Choices[0].Message) + + currLength := estimateTokenCount(messages) + + if currLength > 3000 { + fmt.Println("Token count exceeded 3000, summarizing context") + summarized, err := summarizeChatSoFar(messages) + if err != nil { + fmt.Printf("error summarizing chat so far | %v\n", err) + continue + } else { + fmt.Printf("Summarized: %v\n", summarized) + } + // reset messages to the system prompt and the summarized prompt + messages = []openai.ChatCompletionMessage{ + { + Role: openai.ChatMessageRoleSystem, + Content: systemPrompt, + }, + { + Role: openai.ChatMessageRoleAssistant, + Content: summarized, + }, + } + } } } + +func estimateTokenCount(messages []openai.ChatCompletionMessage) int { + messagesRaw, err := json.Marshal(messages) + if err != nil { + fmt.Printf("error marshalling messages for token size estimation | %v\n", err) + return 0 + } + + length, err := LLMMapper.GetTokenCount(string(messagesRaw)) + if err != nil { + fmt.Printf("error getting token count | %v\n", err) + return 0 + } + fmt.Printf("Token Count: %v\n", length) + return length +} + +func summarizeChatSoFar(messages []openai.ChatCompletionMessage) (string, error) { + messagesRaw, err := json.Marshal(messages) + if err != nil { + return "", fmt.Errorf("error marshalling messages for token size estimation | %w", err) + } + + summarizeConvoPrompt, err := promptBuilder.BuildPrompt("summarize.tmpl", struct{ WordLimit int }{WordLimit: 100}) + if err != nil { + return "", fmt.Errorf("error building prompt for summarization | %w", err) + } + + resp, err := LLMMapper.SendPrompt([]openai.ChatCompletionMessage{ + { + Role: openai.ChatMessageRoleSystem, + Content: summarizeConvoPrompt, + }, + { + Role: openai.ChatMessageRoleUser, + Content: string(messagesRaw), + }, + }) + if err != nil { + return "", fmt.Errorf("error summarizing conversation | %w", err) + } + + if len(resp.Choices) == 0 { + return "", fmt.Errorf("no choices returned for summarization") + } + + return resp.Choices[0].Message.Content, nil +} + +// TODO: anything to be stored in the database should be chunked to sizes between 512 and 1024 tokens +// it should also overlap with the previous chunk by 100-200 tokens +// When the LLM asks for more context, it should be able to use the database to find the most relevant chunks here is how: +// We will get the embeddings for each prompt and use those embeddings to search for the closest 6 chunks +// we will use a separate LLM prompt to make an attempt to select and sort the chunks based on the user's input +// then we will add the best matched chunks to the main prompt as further context for the given prompt diff --git a/main.go b/main.go index 85359ea..850b3a8 100644 --- a/main.go +++ b/main.go @@ -1,22 +1,13 @@ package main import ( - "bytes" "context" + "ctxGPT/LLMMapper" "ctxGPT/database" + "ctxGPT/promptBuilder" "fmt" - "os" - "strings" - "text/template" - "time" - - "github.com/pkoukk/tiktoken-go" - "github.com/sashabaranov/go-openai" ) -const encodingName = "gpt-4" -const model = openai.GPT4TurboPreview - func main() { db, err := database.NewDB() @@ -38,88 +29,14 @@ func main() { // to get text out of PDF, DOC, DOCX, XML, HTML, RTF, ODT pages documents and images to plain text // use https://github.com/sajari/docconv - summarizeConvoPrompt, err := BuildPrompt("summarize.tmpl", struct{ WordLimit int }{WordLimit: 100}) + summarizeConvoPrompt, err := promptBuilder.BuildPrompt("summarize.tmpl", struct{ WordLimit int }{WordLimit: 100}) if err != nil { panic(err) } fmt.Println(summarizeConvoPrompt) - tokenCount, err := GetTokenCount(summarizeConvoPrompt) + tokenCount, err := LLMMapper.GetTokenCount(summarizeConvoPrompt) if err != nil { panic(err) } fmt.Println(tokenCount) } - -func BuildPrompt(name string, in interface{}) (string, error) { - fileLocation := "./prompts/" + name - tmpl, err := template.New(name).ParseFiles(fileLocation) - if err != nil { - return "", fmt.Errorf("error parsing template: %w", err) - } - b := bytes.Buffer{} - err = tmpl.Execute(&b, in) - if err != nil { - return "", fmt.Errorf("error executing template: %w", err) - } - return b.String(), nil -} - -func GetTokenCount(input string) (int, error) { - tke, err := tiktoken.EncodingForModel(encodingName) // cached in "TIKTOKEN_CACHE_DIR" - if err != nil { - return 0, fmt.Errorf("error getting encoding: %w", err) - } - token := tke.Encode(input, nil, nil) - return len(token), nil -} - -// SinglePromptInteraction calls openai chat endpoint with just a system prompt and a user prompt and returns the response -func SinglePromptInteraction(systemPrompt, prompt string) (openai.ChatCompletionResponse, error) { - return singlePromptInteraction(systemPrompt, prompt, 5) -} - -// singlePromptInteraction calls openai chat endpoint with just a system prompt and a user prompt and returns the response -// it also attempts 5 retries spaced 5 seconds apart in the case of rate limiting errors -func singlePromptInteraction(systemPrompt, prompt string, retries int) (openai.ChatCompletionResponse, error) { - - client := openai.NewClient(os.Getenv("OPENAI_API_KEY")) - messages := []openai.ChatCompletionMessage{ - { - Role: openai.ChatMessageRoleSystem, - Content: systemPrompt, - }, - { - Role: openai.ChatMessageRoleUser, - Content: prompt, - }, - } - - resp, err := client.CreateChatCompletion( - context.Background(), - openai.ChatCompletionRequest{ - Model: model, // switch to the configured Model - Messages: messages, - MaxTokens: 256, - Temperature: 0, - }, - ) - if err != nil { - // if 429, wait and try again - if strings.Contains(err.Error(), "429") && retries > 0 { - seconds := (1 / retries) * 60 // back off for each retry e.g. 12, 15, 20, 30, 60 - fmt.Printf("429 error, waiting %v seconds...\n", seconds) - time.Sleep(time.Duration(seconds) * time.Second) - return singlePromptInteraction(systemPrompt, prompt, retries-1) // TODO: establish base case to prevent forever retrying - } - return openai.ChatCompletionResponse{}, fmt.Errorf("ChatCompletion request error: %w", err) - } - - return resp, nil -} - -// TODO: anything to be stored in the database should be chunked to sizes between 512 and 1024 tokens -// it should also overlap with the previous chunk by 100-200 tokens -// When the LLM asks for more context, it should be able to use the database to find the most relevant chunks here is how: -// We will get the embeddings for each prompt and use those embeddings to search for the closest 6 chunks -// we will use a separate LLM prompt to make an attempt to select and sort the chunks based on the user's input -// then we will add the best matched chunks to the main prompt as further context for the given prompt diff --git a/promptBuilder/promptBuilder.go b/promptBuilder/promptBuilder.go new file mode 100644 index 0000000..14a9f3a --- /dev/null +++ b/promptBuilder/promptBuilder.go @@ -0,0 +1,21 @@ +package promptBuilder + +import ( + "bytes" + "fmt" + "text/template" +) + +func BuildPrompt(name string, in interface{}) (string, error) { + fileLocation := "./prompts/" + name + tmpl, err := template.New(name).ParseFiles(fileLocation) + if err != nil { + return "", fmt.Errorf("error parsing template: %w", err) + } + b := bytes.Buffer{} + err = tmpl.Execute(&b, in) + if err != nil { + return "", fmt.Errorf("error executing template: %w", err) + } + return b.String(), nil +} diff --git a/transcript.md b/transcript.md new file mode 100644 index 0000000..b02fb7e --- /dev/null +++ b/transcript.md @@ -0,0 +1,128 @@ +You: Hey here. This is my first time trying a chat with you in my terminal. +AI: Welcome! It's great to have you trying out the chat in your terminal. Feel free to ask me questions or tell me what you need assistance with. Whether you're looking for information, technical help, or just a conversation, I' +m here to assist you. How can I help you today? +You: I am building an authentication application that developers can use to make their applications more secure. What do you think I should do in order to engender trust in the my application so they'll try it out? +AI: When building an authentication application that developers will trust and want to use, several key factors come into play. Here's a list of steps and strategies you can employ to build trust and encourage adoption: + +1. **Security First**: Prioritize strong and up-to-date security practices within your application. Use the latest encryption standards, secure coding practices, and regularly audit your codebase for security vulnerabilities. + +2. **Transparency**: Be transparent about your security practices. Publish your security measures, data handling policies, and compliance certifications if any. Openness about how you manage and protect user data can build signif +icant trust. + +3. **Compliance**: Ensure that your application complies with relevant industry standards and regulations like GDPR, HIPAA, CCPA, and other data protection laws applicable to your users. + +4. **Third-party Audits and Certifications**: Obtain audits from reputable security firms and pursue industry-recognized certifications to validate your security claims. + +5. **Clear Documentation**: Provide comprehensive, clear, and accessible documentation for developers. Documentation should include integration guides, API references, best practices, and support resources. + +6. **Developer Support**: Offer responsive and helpful support. This can include support tickets, community forums, chat support, and documentation to help resolve any issues quickly. + +7. ** +You: continue +AI: Open Source: If it's feasible for your business model, consider open-sourcing at least parts of your application. This allows developers to inspect the code themselves for security practices, which can significantly increase + trust. Transparency in your development process is often well-received within the developer community. + +8. **Case Studies and Testimonials**: Share experiences of other developers and companies that have successfully integrated your authentication application. Real-world examples and endorsements can be powerful in demonstrating th +e reliability and benefits of your product. + +9. **Free Tier or Trial Period**: Offering a free tier or a trial period can encourage developers to test your application without any financial commitment. This can help alleviate initial hesitation and allows them to directly a +ssess the quality and integration process. + +10. **Easy Integration**: Make the integration process as straightforward as possible. The easier it is to use your application, the more likely developers will try it and stick with it. + +11. **Marketing and Thought Leadership**: Create valuable content for developers – blog posts, whitepapers, webinars, and tutorials – that position you as a thought leader in authentication and security. Show your expertise and s +hare knowledge that can help developers beyond just using your service. + +12. **Community Engagement**: Engage with the developer community through social media, tech conferences, meetups +You: continue +AI: , and hackathons. Being actively involved and visible in the community can help build your reputation and trust. Moreover, listening to feedback and incorporating it into your product's development demonstrates that you valu +e user input and are committed to continuous improvement. + +13. **Partnerships**: Form partnerships with other trusted companies and organizations in the tech field. This not just extends your reach but also confers upon you a part of the trust that these entities have already established +. + +14. **Reliability**: Ensure your authentication service is highly available and performs consistently. Downtime or performance issues can erode trust quickly, so invest in robust infrastructure and have a solid incident response +plan. + +15. **Updates and Communication**: Keep your product up-to-date with the latest security trends and threats. Communicate proactively with your user base about new features, patches, and updates. + +16. **User Authentication Options**: Provide various authentication methods such as multi-factor authentication, single sign-on, biometric verification, etc. Flexibility can make your application more appealing to a broader audie +nce. + +17. **Privacy Policy**: Have a clear and concise privacy policy explaining how you handle and protect user data. Privacy is a major concern for many users and being upfront about it can aid in building trust. + +18. ** +You: it seems like you might have more +AI: Business and Pricing Model Clarity: Transparently explain your pricing model and business practices to avoid any hidden charges or surprises. Trust can be quickly eroded if customers feel they've been misled about the cost o +r terms of service. + +19. **User Control and Consent**: Empower users with control over their data and require explicit consent where needed. When users feel in control of their data, they are more likely to trust your application. + +20. **Feedback Loops**: Implement a system for users to provide feedback easily and make it clear that their input is valued and considered. This not only helps you improve your service but also demonstrates to users that you are + responsive and customer-centric. + +21. **Bounty Programs**: Consider implementing a bug bounty program to encourage ethical hackers to report vulnerabilities in your system. This can help you identify and fix security issues before they are exploited and shows you +r commitment to security. + +22. **Incremental Rollouts**: If viable, roll out your application incrementally, starting with beta testers or a limited audience. This allows you to gather initial feedback and make necessary improvements, thereby refining your + product before a full-scale launch. + +Trust is essential in the world of security and authentication. By following the above strategies and maintaining a clear focus on quality and transparency, you can +You: continue once more +AI: build a strong foundation for trust with your customer base. Here are a few more things you can consider to engender trust in your authentication application: + +23. **User Education and Resources**: Educate your users about security best practices and the importance of strong authentication. Provide resources such as guides and articles that explain security concepts in layman's terms. T +his can also involve training materials or webinars that teach users how to get the most out of your application. + +24. **Data Breach Response Plan**: Have a clear and tested data breach response plan in place. Being able to react swiftly and transparently in the event of a breach can help maintain trust. It shows customers that you are prepar +ed and proactive in dealing with worst-case scenarios. + +25. **Responsible Disclosure**: Encourage responsible disclosure of security issues by stating your policies clearly, including how you handle reports of security vulnerabilities, and possibly offering recognition or rewards for +those reports. + +26. **Regular Security Updates and Patch Notes**: Regularly update your application with security patches and provide detailed notes on what was improved. Keeping your system up-to-date and informing users about changes can insti +ll confidence in your commitment to their security. + +Remember, building trust is a long-term effort that involves consistent communication, a proactive stance on security, and +You: + + +------------------- Fixed the length issue ---------------------------------- + +Building trust in an authentication application is crucial because security is a top concern for developers and their users. Here are several steps you can take to engender trust in your application: + +1. **Security First Design**: Ensure that your application is designed with security as a primary focus. Follow best practices for secure coding, and regularly audit and update your code to address any vulnerabilities. + +2. **Transparency**: Be open about how your application works. Provide detailed documentation that explains the security measures you have implemented and how they protect your users. + +3. **Third-Party Audits and Certifications**: Obtain security certifications from reputable organizations. Having your application's security audited by a third party can provide an unbiased validation of your security claims. + +4. **Compliance with Standards**: Ensure your application complies with relevant security standards and regulations (e.g., OAuth 2.0, OpenID Connect, GDPR, HIPAA, etc.). Compliance demonstrates that you are serious about security + and privacy. + +5. **Use of Well-Known Algorithms and Protocols**: Use industry-standard encryption algorithms and authentication protocols. Avoid proprietary or obscure methods that have not been thoroughly vetted by the security community. + +6. **Open Source**: Consider making your application open source. This allows the community to review and contribute to your code, which can improve security and trustworthiness. + +7. **Easy Integration**: Make it easy for developers to integrate your application into their systems. Provide SDKs, APIs, and clear documentation. If developers find it difficult to use your application, they are less likely to + trust it. + +8. **Demonstrable Reliability**: Offer a reliable service with high uptime. Reliability is a key component of trust, and if your service is consistently available, developers are more likely to trust it. + +9. **Customer Support**: Provide excellent customer support. Be responsive to inquiries and issues. If developers feel supported, they are more likely to trust your application. + +10. **Case Studies and Testimonials**: Share success stories from other developers who have successfully implemented your application. Testimonials from satisfied users can be very persuasive. + +11. **Free Trial or Freemium Model**: Offer a free trial or a freemium version of your application. This allows developers to test your application and see its benefits firsthand without any financial commitment. + +12. **Marketing and Branding**: Invest in professional marketing and branding. A strong brand can convey trustworthiness and professionalism. + +13. **Community Engagement**: Engage with the developer community through forums, social media, conferences, and hackathons. Building relationships with developers can lead to trust and word-of-mouth recommendations. + +14. **Educational Content**: Provide educational content that helps developers understand security concepts and best practices. This positions you as an expert in the field and builds trust in your application. + +15. **Regular Updates and Communication**: Keep your application up-to-date with the latest security features and communicate these updates to your users. Regular communication about improvements demonstrates your commitment to s + ecurity. + +Remember, building trust takes time and consistent effort. By focusing on security, transparency, and user support, you can create a strong foundation of trust for your authentication application. +