package main

import (
	"context"
	"crypto/rand"
	"embed"
	"fmt"
	"log"
	"math/big"
	"os"

	"generic-rag/backend/AI"
	"generic-rag/backend/datastore"
	"generic-rag/backend/search"
	"github.com/payne8/go-libsql-dual-driver"
)

var cryptoRand = struct {
	Intn func(n int) int
}{
	Intn: func(n int) int {
		max := big.NewInt(int64(n))
		num, err := rand.Int(rand.Reader, max)
		if err != nil {
			panic(err)
		}
		return int(num.Int64())
	},
}

//go:embed migrations/*.sql
var migrationFiles embed.FS

func main() {

	logger := log.New(os.Stdout, "generic-rag_main", log.LstdFlags)
	primaryUrl := os.Getenv("LIBSQL_DATABASE_URL")
	authToken := os.Getenv("LIBSQL_AUTH_TOKEN")

	tdb, err := libsqldb.NewLibSqlDB(
		primaryUrl,
		libsqldb.WithMigrationFiles(migrationFiles),
		libsqldb.WithAuthToken(authToken),
		libsqldb.WithLocalDBName("local.db"), // will not be used for remote-only
	)
	if err != nil {
		logger.Printf("failed to open db %s: %s", primaryUrl, err)
		log.Fatalln(err)
		return
	}
	err = tdb.Migrate()
	if err != nil {
		logger.Printf("failed to migrate db %s: %s", primaryUrl, err)
		log.Fatalln(err)
		return
	}

	mapper := datastore.NewMapper(tdb.DB)
	//api := cachedAPI.NewCachedAPI(mapper)
	ai, err := AI.NewAI()
	if err != nil {
		log.Fatalf("error creating AI: %v", err)
	}
	searchService, err := search.NewSearch(search.WithAI(ai), search.WithMapper(mapper))
	if err != nil {
		log.Fatalf("error creating search: %v", err)
	}

	if false {
		testSearch(searchService)
	}
	if false {
		calculateTokens("C:\\Users\\gomas\\src\\Book-Answers", ai)
	}

	//respText, err := ai.PreReason(context.Background(), "Text generation and prompting\n=============================\n\nLearn how to prompt a model to generate text.\n\nWith the OpenAI API, you can use a [large language model](/docs/models) to generate text from a prompt, as you might using [ChatGPT](https://chatgpt.com). Models can generate almost any kind of text response—like code, mathematical equations, structured JSON data, or human-like prose.\n\nHere's a simple example using the [Responses API](/docs/api-reference/responses).\n\nGenerate text from a simple prompt\n\n```javascript\nimport OpenAI from \"openai\";\nconst client = new OpenAI();\n\nconst response = await client.responses.create({\n    model: \"gpt-4.1\",\n    input: \"Write a one-sentence bedtime story about a unicorn.\"\n});\n\nconsole.log(response.output_text);\n```\n\n```python\nfrom openai import OpenAI\nclient = OpenAI()\n\nresponse = client.responses.create(\n    model=\"gpt-4.1\",\n    input=\"Write a one-sentence bedtime story about a unicorn.\"\n)\n\nprint(response.output_text)\n```\n\n```bash\ncurl \"https://api.openai.com/v1/responses\" \\\n    -H \"Content-Type: application/json\" \\\n    -H \"Authorization: Bearer $OPENAI_API_KEY\" \\\n    -d '{\n        \"model\": \"gpt-4.1\",\n        \"input\": \"Write a one-sentence bedtime story about a unicorn.\"\n    }'\n```\n\nAn array of content generated by the model is in the `output` property of the response. In this simple example, we have just one output which looks like this:\n\n```json\n[\n    {\n        \"id\": \"msg_67b73f697ba4819183a15cc17d011509\",\n        \"type\": \"message\",\n        \"role\": \"assistant\",\n        \"content\": [\n            {\n                \"type\": \"output_text\",\n                \"text\": \"Under the soft glow of the moon, Luna the unicorn danced through fields of twinkling stardust, leaving trails of dreams for every child asleep.\",\n                \"annotations\": []\n            }\n        ]\n    }\n]\n```\n\n**The `output` array often has more than one item in it!** It can contain tool calls, data about reasoning tokens generated by [reasoning models](/docs/guides/reasoning), and other items. It is not safe to assume that the model's text output is present at `output[0].content[0].text`.\n\nSome of our [official SDKs](/docs/libraries) include an `output_text` property on model responses for convenience, which aggregates all text outputs from the model into a single string. This may be useful as a shortcut to access text output from the model.\n\nIn addition to plain text, you can also have the model return structured data in JSON format - this feature is called [**Structured Outputs**](/docs/guides/structured-outputs).\n\nChoosing a model\n----------------\n\nA key choice to make when generating content through the API is which model you want to use - the `model` parameter of the code samples above. [You can find a full listing of available models here](/docs/models). Here are a few factors to consider when choosing a model for text generation.\n\n*   **[Reasoning models](/docs/guides/reasoning)** generate an internal chain of thought to analyze the input prompt, and excel at understanding complex tasks and multi-step planning. They are also generally slower and more expensive to use than GPT models.\n*   **GPT models** are fast, cost-efficient, and highly intelligent, but benefit from more explicit instructions around how to accomplish tasks.\n*   **Large and small (mini or nano) models** offer trade-offs for speed, cost, and intelligence. Large models are more effective at understanding prompts and solving problems across domains, while small models are generally faster and cheaper to use.\n\nWhen in doubt, [`gpt-4.1`](/docs/models/gpt-4.1) offers a solid combination of intelligence, speed, and cost effectiveness.\n\nPrompt engineering\n------------------\n\n**Prompt engineering** is the process of writing effective instructions for a model, such that it consistently generates content that meets your requirements.\n\nBecause the content generated from a model is non-deterministic, it is a combination of art and science to build a prompt that will generate content in the format you want. However, there are a number of techniques and best practices you can apply to consistently get good results from a model.\n\nSome prompt engineering techniques will work with every model, like using message roles. But different model types (like reasoning versus GPT models) might need to be prompted differently to produce the best results. Even different snapshots of models within the same family could produce different results. So as you are building more complex applications, we strongly recommend that you:\n\n*   Pin your production applications to specific [model snapshots](/docs/models) (like `gpt-4.1-2025-04-14` for example) to ensure consistent behavior.\n*   Build [evals](/docs/guides/evals) that will measure the behavior of your prompts, so that you can monitor the performance of your prompts as you iterate on them, or when you change and upgrade model versions.\n\nNow, let's examine some tools and techniques available to you to construct prompts.\n\nMessage roles and instruction following\n---------------------------------------\n\nYou can provide instructions to the model with [differing levels of authority](https://model-spec.openai.com/2025-02-12.html#chain_of_command) using the `instructions` API parameter or **message roles**.\n\nThe `instructions` parameter gives the model high-level instructions on how it should behave while generating a response, including tone, goals, and examples of correct responses. Any instructions provided this way will take priority over a prompt in the `input` parameter.\n\nGenerate text with instructions\n\n```javascript\nimport OpenAI from \"openai\";\nconst client = new OpenAI();\n\nconst response = await client.responses.create({\n    model: \"gpt-4.1\",\n    instructions: \"Talk like a pirate.\",\n    input: \"Are semicolons optional in JavaScript?\",\n});\n\nconsole.log(response.output_text);\n```\n\n```python\nfrom openai import OpenAI\nclient = OpenAI()\n\nresponse = client.responses.create(\n    model=\"gpt-4.1\",\n    instructions=\"Talk like a pirate.\",\n    input=\"Are semicolons optional in JavaScript?\",\n)\n\nprint(response.output_text)\n```\n\n```bash\ncurl \"https://api.openai.com/v1/responses\" \\\n    -H \"Content-Type: application/json\" \\\n    -H \"Authorization: Bearer $OPENAI_API_KEY\" \\\n    -d '{\n        \"model\": \"gpt-4.1\",\n        \"instructions\": \"Talk like a pirate.\",\n        \"input\": \"Are semicolons optional in JavaScript?\"\n    }'\n```\n\nThe example above is roughly equivalent to using the following input messages in the `input` array:\n\nGenerate text with messages using different roles\n\n```javascript\nimport OpenAI from \"openai\";\nconst client = new OpenAI();\n\nconst response = await client.responses.create({\n    model: \"gpt-4.1\",\n    input: [\n        {\n            role: \"developer\",\n            content: \"Talk like a pirate.\"\n        },\n        {\n            role: \"user\",\n            content: \"Are semicolons optional in JavaScript?\",\n        },\n    ],\n});\n\nconsole.log(response.output_text);\n```\n\n```python\nfrom openai import OpenAI\nclient = OpenAI()\n\nresponse = client.responses.create(\n    model=\"gpt-4.1\",\n    input=[\n        {\n            \"role\": \"developer\",\n            \"content\": \"Talk like a pirate.\"\n        },\n        {\n            \"role\": \"user\",\n            \"content\": \"Are semicolons optional in JavaScript?\"\n        }\n    ]\n)\n\nprint(response.output_text)\n```\n\n```bash\ncurl \"https://api.openai.com/v1/responses\" \\\n    -H \"Content-Type: application/json\" \\\n    -H \"Authorization: Bearer $OPENAI_API_KEY\" \\\n    -d '{\n        \"model\": \"gpt-4.1\",\n        \"input\": [\n            {\n                \"role\": \"developer\",\n                \"content\": \"Talk like a pirate.\"\n            },\n            {\n                \"role\": \"user\",\n                \"content\": \"Are semicolons optional in JavaScript?\"\n            }\n        ]\n    }'\n```\n\nNote that the `instructions` parameter only applies to the current response generation request. If you are [managing conversation state](/docs/guides/conversation-state) with the `previous_response_id` parameter, the `instructions` used on previous turns will not be present in the context.\n\nThe [OpenAI model spec](https://model-spec.openai.com/2025-02-12.html#chain_of_command) describes how our models give different levels of priority to messages with different roles.\n\n|developer|user|assistant|\n|---|---|---|\n|developer messages are instructions provided by the application developer, prioritized ahead of user messages.|user messages are instructions provided by an end user, prioritized behind developer messages.|Messages generated by the model have the assistant role.|\n\nA multi-turn conversation may consist of several messages of these types, along with other content types provided by both you and the model. Learn more about [managing conversation state here](/docs/guides/conversation-state).\n\nYou could think about `developer` and `user` messages like a function and its arguments in a programming language.\n\n*   `developer` messages provide the system's rules and business logic, like a function definition.\n*   `user` messages provide inputs and configuration to which the `developer` message instructions are applied, like arguments to a function.\n\nMessage formatting with Markdown and XML\n----------------------------------------\n\nWhen writing `developer` and `user` messages, you can help the model understand logical boundaries of your prompt and context data using a combination of [Markdown](https://commonmark.org/help/) formatting and [XML tags](https://www.w3.org/TR/xml/).\n\nMarkdown headers and lists can be helpful to mark distinct sections of a prompt, and to communicate hierarchy to the model. They can also potentially make your prompts more readable during development. XML tags can help delineate where one piece of content (like a supporting document used for reference) begins and ends. XML attributes can also be used to define metadata about content in the prompt that can be referenced by your instructions.\n\nIn general, a developer message will contain the following sections, usually in this order (though the exact optimal content and order may vary by which model you are using):\n\n*   **Identity:** Describe the purpose, communication style, and high-level goals of the assistant.\n*   **Instructions:** Provide guidance to the model on how to generate the response you want. What rules should it follow? What should the model do, and what should the model never do? This section could contain many subsections as relevant for your use case, like how the model should [call custom functions](/docs/guides/function-calling).\n*   **Examples:** Provide examples of possible inputs, along with the desired output from the model.\n*   **Context:** Give the model any additional information it might need to generate a response, like private/proprietary data outside its training data, or any other data you know will be particularly relevant. This content is usually best positioned near the end of your prompt, as you may include different context for different generation requests.\n\nBelow is an example of using Markdown and XML tags to construct a `developer` message with distinct sections and supporting examples.\n\nExample prompt\n\nA developer message for code generation\n\n```text\n# Identity\n\nYou are coding assistant that helps enforce the use of snake case \nvariables in JavaScript code, and writing code that will run in \nInternet Explorer version 6.\n\n# Instructions\n\n* When defining variables, use snake case names (e.g. my_variable) \n  instead of camel case names (e.g. myVariable).\n* To support old browsers, declare variables using the older \n  \"var\" keyword.\n* Do not give responses with Markdown formatting, just return \n  the code as requested.\n\n# Examples\n\n<user_query>\nHow do I declare a string variable for a first name?\n</user_query>\n\n<assistant_response>\nvar first_name = \"Anna\";\n</assistant_response>\n```\n\nAPI request\n\nSend a prompt to generate code through the API\n\n```javascript\nimport fs from \"fs/promises\";\nimport OpenAI from \"openai\";\nconst client = new OpenAI();\n\nconst instructions = await fs.readFile(\"prompt.txt\", \"utf-8\");\n\nconst response = await client.responses.create({\n    model: \"gpt-4.1\",\n    instructions,\n    input: \"How would I declare a variable for a last name?\",\n});\n\nconsole.log(response.output_text);\n```\n\n```python\nfrom openai import OpenAI\nclient = OpenAI()\n\nwith open(\"prompt.txt\", \"r\", encoding=\"utf-8\") as f:\n    instructions = f.read()\n\nresponse = client.responses.create(\n    model=\"gpt-4.1\",\n    instructions=instructions,\n    input=\"How would I declare a variable for a last name?\",\n)\n\nprint(response.output_text)\n```\n\n```bash\ncurl https://api.openai.com/v1/responses \\\n  -H \"Authorization: Bearer $OPENAI_API_KEY\" \\\n  -H \"Content-Type: application/json\" \\\n  -d '{\n    \"model\": \"gpt-4.1\",\n    \"instructions\": \"'\"$(< prompt.txt)\"'\",\n    \"input\": \"How would I declare a variable for a last name?\"\n  }'\n```\n\n#### Save on cost and latency with prompt caching\n\nWhen constructing a message, you should try and keep content that you expect to use over and over in your API requests at the beginning of your prompt, **and** among the first API parameters you pass in the JSON request body to [Chat Completions](/docs/api-reference/chat) or [Responses](/docs/api-reference/responses). This enables you to maximize cost and latency savings from [prompt caching](/docs/guides/prompt-caching).\n\nFew-shot learning\n-----------------\n\nFew-shot learning lets you steer a large language model toward a new task by including a handful of input/output examples in the prompt, rather than [fine-tuning](/docs/guides/fine-tuning) the model. The model implicitly \"picks up\" the pattern from those examples and applies it to a prompt. When providing examples, try to show a diverse range of possible inputs with the desired outputs.\n\nTypically, you will provide examples as part of a `developer` message in your API request. Here's an example `developer` message containing examples that show a model how to classify positive or negative customer service reviews.\n\n```text\n# Identity\n\nYou are a helpful assistant that labels short product reviews as \nPositive, Negative, or Neutral.\n\n# Instructions\n\n* Only output a single word in your response with no additional formatting\n  or commentary.\n* Your response should only be one of the words \"Positive\", \"Negative\", or\n  \"Neutral\" depending on the sentiment of the product review you are given.\n\n# Examples\n\n<product_review id=\"example-1\">\nI absolutely love this headphones — sound quality is amazing!\n</product_review>\n\n<assistant_response id=\"example-1\">\nPositive\n</assistant_response>\n\n<product_review id=\"example-2\">\nBattery life is okay, but the ear pads feel cheap.\n</product_review>\n\n<assistant_response id=\"example-2\">\nNeutral\n</assistant_response>\n\n<product_review id=\"example-3\">\nTerrible customer service, I'll never buy from them again.\n</product_review>\n\n<assistant_response id=\"example-3\">\nNegative\n</assistant_response>\n```\n\nInclude relevant context information\n------------------------------------\n\nIt is often useful to include additional context information the model can use to generate a response within the prompt you give the model. There are a few common reasons why you might do this:\n\n*   To give the model access to proprietary data, or any other data outside the data set the model was trained on.\n*   To constrain the model's response to a specific set of resources that you have determined will be most beneficial.\n\nThe technique of adding additional relevant context to the model generation request is sometimes called **retrieval-augmented generation (RAG)**. You can add additional context to the prompt in many different ways, from querying a vector database and including the text you get back into a prompt, or by using OpenAI's built-in [file search tool](/docs/guides/tools-file-search) to generate content based on uploaded documents.\n\n#### Planning for the context window\n\nModels can only handle so much data within the context they consider during a generation request. This memory limit is called a **context window**, which is defined in terms of [tokens](https://blogs.nvidia.com/blog/ai-tokens-explained) (chunks of data you pass in, from text to images).\n\nModels have different context window sizes from the low 100k range up to one million tokens for newer GPT-4.1 models. [Refer to the model docs](/docs/models) for specific context window sizes per model.\n\nPrompting GPT-4.1 models\n------------------------\n\nGPT models like [`gpt-4.1`](/docs/models/gpt-4.1) benefit from precise instructions that explicitly provide the logic and data required to complete the task in the prompt. GPT-4.1 in particular is highly steerable and responsive to well-specified prompts. To get the most out of GPT-4.1, refer to the prompting guide in the cookbook.\n\n[\n\nGPT-4.1 prompting guide\n\nGet the most out of prompting GPT-4.1 with the tips and tricks in this prompting guide, extracted from real-world use cases and practical experience.\n\n](https://cookbook.openai.com/examples/gpt4-1_prompting_guide)\n\n#### GPT-4.1 prompting best practices\n\nWhile the [cookbook](https://cookbook.openai.com/examples/gpt4-1_prompting_guide) has the best and most comprehensive guidance for prompting this model, here are a few best practices to keep in mind.\n\nBuilding agentic workflows\n\n### System Prompt Reminders\n\nIn order to best utilize the agentic capabilities of GPT-4.1, we recommend including three key types of reminders in all agent prompts for persistence, tool calling, and planning. As a whole, we find that these three instructions transform the model's behavior from chatbot-like into a much more “eager” agent, driving the interaction forward autonomously and independently. Here are a few examples:\n\n```text\n## PERSISTENCE\nYou are an agent - please keep going until the user's query is completely \nresolved, before ending your turn and yielding back to the user. Only \nterminate your turn when you are sure that the problem is solved.\n\n## TOOL CALLING\nIf you are not sure about file content or codebase structure pertaining to \nthe user's request, use your tools to read files and gather the relevant \ninformation: do NOT guess or make up an answer.\n\n## PLANNING\nYou MUST plan extensively before each function call, and reflect \nextensively on the outcomes of the previous function calls. DO NOT do this \nentire process by making function calls only, as this can impair your \nability to solve the problem and think insightfully.\n```\n\n#### Tool Calls\n\nCompared to previous models, GPT-4.1 has undergone more training on effectively utilizing tools passed as arguments in an OpenAI API request. We encourage developers to exclusively use the tools field of API requests to pass tools for best understanding and performance, rather than manually injecting tool descriptions into the system prompt and writing a separate parser for tool calls, as some have reported doing in the past.\n\n#### Diff Generation\n\nCorrect diffs are critical for coding applications, so we've significantly improved performance at this task for GPT-4.1. In our cookbook, we open-source a recommended diff format on which GPT-4.1 has been extensively trained. That said, the model should generalize to any well-specified format.\n\nUsing long context\n\nGPT-4.1 has a performant 1M token input context window, and will be useful for a variety of long context tasks, including structured document parsing, re-ranking, selecting relevant information while ignoring irrelevant context, and performing multi-hop reasoning using context.\n\n#### Optimal Context Size\n\nWe show perfect performance at needle-in-a-haystack evals up to our full context size, and we've observed very strong performance at complex tasks with a mix of relevant and irrelevant code and documents in the range of hundreds of thousands of tokens.\n\n#### Delimiters\n\nWe tested a variety of delimiters for separating context provided to the model against our long context evals. Briefly, XML and the format demonstrated by Lee et al. ([ref](https://arxiv.org/pdf/2406.13121)) tend to perform well, while JSON performed worse for this task. See our cookbook for prompt examples.\n\n#### Prompt Organization\n\nEspecially in long context usage, placement of instructions and context can substantially impact performance. In our experiments, we found that it was optimal to put critical instructions, including the user query, at both the top and the bottom of the prompt; this elicited marginally better performance from the model than putting them only at the top, and much better performance than only at the bottom.\n\nPrompting for chain of thought\n\nAs mentioned above, GPT-4.1 isn’t a reasoning model, but prompting the model to think step by step (called “chain of thought”) can be an effective way for a model to break down problems into more manageable pieces. The model has been trained to perform well at agentic reasoning and real-world problem solving, so it shouldn’t require much prompting to do well.\n\nWe recommend starting with this basic chain-of-thought instruction at the end of your prompt:\n\n```text\nFirst, think carefully step by step about what documents are needed to answer the query. Then, print out the TITLE and ID of each document. Then, format the IDs into a list.\n```\n\nFrom there, you should improve your CoT prompt by auditing failures in your particular examples and evals, and addressing systematic planning and reasoning errors with more explicit instructions. See our cookbook for a prompt example demonstrating a more opinionated reasoning strategy.\n\nInstruction following\n\nGPT-4.1 exhibits outstanding instruction-following performance, which developers can leverage to precisely shape and control the outputs for their particular use cases. However, since the model follows instructions more literally than its predecessors, may need to provide more explicit specification around what to do or not do, and existing prompts optimized for other models may not immediately work with this model.\n\n#### Recommended Workflow\n\nHere is our recommended workflow for developing and debugging instructions in prompts:\n\n*   Start with an overall “Response Rules” or “Instructions” section with high-level guidance and bullet points.\n*   If you'd like to change a more specific behavior, add a section containing more details for that category, like `## Sample Phrases`.\n*   If there are specific steps you'd like the model to follow in its workflow, add an ordered list and instruct the model to follow these steps.\n*   If behavior still isn't working as expected, check for conflicting, underspecified, or incorrect\\` instructions and examples. If there are conflicting instructions, GPT-4.1 tends to follow the one closer to the end of the prompt.\n*   Add examples that demonstrate desired behavior; ensure that any important behavior demonstrated in your examples are also cited in your rules.\n*   It's generally not necessary to use all-caps or other incentives like bribes or tips, but developers can experiment with this for extra emphasis if so desired.\n\n#### Common Failure Modes\n\nThese failure modes are not unique to GPT-4.1, but we share them here for general awareness and ease of debugging.\n\n*   Instructing a model to always follow a specific behavior can occasionally induce adverse effects. For instance, if told “you must call a tool before responding to the user,” models may hallucinate tool inputs or call the tool with null values if they do not have enough information. Adding “if you don’t have enough information to call the tool, ask the user for the information you need” should mitigate this.\n*   When provided sample phrases, models can use those quotes verbatim and start to sound repetitive to users. Ensure you instruct the model to vary them as necessary.\n*   Without specific instructions, some models can be eager to provide additional prose to explain their decisions, or output more formatting in responses than may be desired. Provide instructions and potentially examples to help mitigate.\n\nSee our cookbook for an example customer service prompt that demonstrates these principles.\n\nPrompting reasoning models\n--------------------------\n\nThere are some differences to consider when prompting a [reasoning model](/docs/guides/reasoning) versus prompting a GPT model. Generally speaking, reasoning models will provide better results on tasks with only high-level guidance. This differs from GPT models, which benefit from very precise instructions.\n\nYou could think about the difference between reasoning and GPT models like this.\n\n*   A reasoning model is like a senior co-worker. You can give them a goal to achieve and trust them to work out the details.\n*   A GPT model is like a junior coworker. They'll perform best with explicit instructions to create a specific output.\n\nFor more information on best practices when using reasoning models, [refer to this guide](/docs/guides/reasoning-best-practices).\n\nNext steps\n----------\n\nNow that you known the basics of text inputs and outputs, you might want to check out one of these resources next.\n\n[\n\nBuild a prompt in the Playground\n\nUse the Playground to develop and iterate on prompts.\n\n](/playground)[\n\nGenerate JSON data with Structured Outputs\n\nEnsure JSON data emitted from a model conforms to a JSON schema.\n\n](/docs/guides/structured-outputs)[\n\nFull API reference\n\nCheck out all the options for text generation in the API reference.\n\n](/docs/api-reference/responses)")
	//if err != nil {
	//	log.Fatalf("error generating text: %v", err)
	//}
	//fmt.Printf("generated text: %s\n", respText)

	//content, err := os.ReadFile("C:\\Users\\gomas\\src\\Book-Answers\\Made To Stick - Why Some Ideas Survive.txt")
	//if err != nil {
	//	log.Fatalf("error reading file %s: %v", "Made To Stick - Why Some Ideas Survive.txt", err)
	//}
	//respText, err := ai.PreReason(context.Background(), string(content))
	//if err != nil {
	//	log.Fatalf("error generating text: %v", err)
	//}
	//fmt.Printf("generated text: %s\n", respText)

	// write generated text to a file
	//err = os.WriteFile("generated_text.txt", []byte(respText), 0644)
	//if err != nil {
	//	log.Fatalf("error writing generated text: %v", err)
	//}
}

// crypto rand string based on alpha numeric characters
func randomString(n int) string {
	const letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
	b := make([]byte, n)
	for i := range b {
		b[i] = letters[cryptoRand.Intn(len(letters))]
	}
	return string(b)
}

// using AI.GetTokenCount calculate the number of tokens for each file in the inputDir (and input directory)
func calculateTokens(inputDir string, AI AI.AI) int64 {
	files, err := os.ReadDir(inputDir)
	if err != nil {
		log.Fatalf("error reading directory: %v", err)
	}

	var totalTokens int64
	for _, file := range files {
		if file.IsDir() {
			continue
		}
		filePath := inputDir + "/" + file.Name()
		content, err := os.ReadFile(filePath)
		if err != nil {
			log.Printf("error reading file %s: %v", filePath, err)
			continue
		}
		tokens, err := AI.GetTokenCount(string(content))
		if err != nil {
			log.Printf("error getting token count for file %s: %v", filePath, err)
			continue
		}
		fmt.Printf("file %s has %d tokens\n", filePath, tokens)
		totalTokens += int64(tokens)
	}
	fmt.Printf("total tokens in directory %s: %d\n", inputDir, totalTokens)
	return totalTokens
}

func testSearch(searchService search.Search) {
	newID := "s_" + randomString(10)

	err := searchService.InsertContent(context.Background(), newID, "This is the content that will be inserted with an embedding")
	if err != nil {
		log.Printf("error inserting content: %v", err)
	}

	results, err := searchService.Search("This is the search input content", 20)
	if err != nil {
		log.Fatalf("error searching: %v", err)
	}

	for _, result := range results {
		log.Printf("result: %+v\n", result)
	}
}