10 分钟打造基于 ChatGPT 的 Markdown 智能文档

OPENAI_API_KEY="<YOUR KEY>"EMBEDBASE_API_KEY="<YOUR KEY>"

# we won't use "pnpm" here, rather the traditional "npm"rm pnpm-lock.yamlnpm inpm run dev

const glob = require("glob");const fs = require("fs");const sync = async () => { // 1. read all files under pages/* with .mdx extension // for each file, read the content const documents = glob.sync("pages/**/*.mdx").map((path) => ({  // we use as id /{pagename} which could be useful to  // provide links in the UI  id: path.replace("pages/", "/").replace("index.mdx", "").replace(".mdx", ""),  // content of the file  data: fs.readFileSync(path, "utf-8") }));  // 2. here we split the documents in chunks, you can do it in many different ways, pick the one you prefer // split documents into chunks of 100 lines const chunks = []; documents.forEach((document) => {  const lines = document.data.split("\n");  const chunkSize = 100;  for (let i = 0; i < lines.length; i += chunkSize) {   const chunk = lines.slice(i, i + chunkSize).join("\n");    chunks.push({      data: chunk   });  } });}sync();

const fetch = require("node-fetch");// your Embedbase api keyconst apiKey = process.env.EMBEDBASE_API_KEY;const sync = async () => { // ... // 3. we then insert the data in Embedbase const response = await fetch("https://embedbase-hosted-usx5gpslaq-uc.a.run.app/v1/documentation", { // "documentation" is your dataset ID  method: "POST",  headers: {   "Authorization": "Bearer " + apiKey,   "Content-Type": "application/json"  },  body: JSON.stringify({   documents: chunks  }) }); const data = await response.json(); console.log(data);}sync();

EMBEDBASE_API_KEY="<YOUR API KEY>" node scripts/sync.js

// update the importsimport { DocsThemeConfig, useTheme } from 'nextra-theme-docs'const Modal = ({ children, open, onClose }) => {  const theme = useTheme();  if (!open) return null;  return (    <div      style={{        position: 'fixed',        top: 0,        left: 0,        right: 0,        bottom: 0,        backgroundColor: 'rgba(0,0,0,0.5)',        zIndex: 100,       }}      onClick={onClose}>            <div        style={{          position: 'absolute',          top: '50%',          left: '50%',          transform: 'translate(-50%, -50%)',          backgroundColor: theme.resolvedTheme === 'dark' ? '#1a1a1a' : 'white',          padding: 20,          borderRadius: 5,          width: '80%',          maxWidth: 700,          maxHeight: '80%',          overflow: 'auto',        }}        onClick={(e) => e.stopPropagation()}>                {children}      </div>        </div>  );};

// update the importsimport React, { useState } from 'react'// we create a Search componentconst Search = () => {  const [open, setOpen] = useState(false);  const [question, setQuestion] = useState("");  // ...  // All the logic that we will see later  const answerQuestion = () => {  }  // ...  return (    <>      <input        placeholder="Ask a question"  // We open the modal here  // to let the user ask a question  onClick={() => setOpen(true)}  type="text"      />      <Modal open={open} onClose={() => setOpen(false)}>        <form onSubmit={answerQuestion} className="nx-flex nx-gap-3">    <input      placeholder="Ask a question"      type="text"      value={question}            onChange={(e) => setQuestion(e.target.value)}          />    <button type="submit">                Ask    </button>        </form>      </Modal>    </>  );}

const config: DocsThemeConfig = {  logo: <span>My Project</span>,  project: {    link: 'https://github.com/shuding/nextra-docs-template',  },  chat: {    link: 'https://discord.com',  },  docsRepositoryBase: 'https://github.com/shuding/nextra-docs-template',  footer: {    text: 'Nextra Docs Template',  },  // add this to use our Search component  search: {    component: <Search />  }}

// pages/api/buildPrompt.tsimport { get_encoding } from "@dqbd/tiktoken";// Load the tokenizer which is designed to work with the embedding modelconst enc = get_encoding('cl100k_base');const apiKey = process.env.EMBEDBASE_API_KEY;// this is how you search Embedbase with a string queryconst search = async (query: string) => { return fetch("https://embedbase-hosted-usx5gpslaq-uc.a.run.app/v1/documentation/search", {  method: "POST",  headers: {   Authorization: "Bearer " + apiKey,   "Content-Type": "application/json"  },  body: JSON.stringify({   query: query  }) }).then(response => response.json());};const createContext = async (question: string, maxLen = 1800) => { // get the similar data to our query from the database const searchResponse = await search(question); let curLen = 0; const returns = []; // We want to add context to some limit of length (tokens) // because usually LLM have limited input size for (const similarity of searchResponse["similarities"]) {  const sentence = similarity["data"];  // count the tokens  const nTokens = enc.encode(sentence).length;  // a token is roughly 4 characters, to learn more  // https://help.openai.com/en/articles/4936856-what-are-tokens-and-how-to-count-them  curLen += nTokens + 4;  if (curLen > maxLen) {   break;  }  returns.push(sentence); } // we join the entries we found with a separator to show it's different return returns.join("\n\n###\n\n");}// this is the endpoint that returns an answer to the clientexport default async function buildPrompt(req, res) { const prompt = req.body.prompt; const context = await createContext(prompt); const newPrompt = `Answer the question based on the context below, and if the question can't be answered based on the context, say "I don't know"\n\nContext: ${context}\n\n---\n\nQuestion: ${prompt}\nAnswer:`; res.status(200).json({ prompt: newPrompt });}

import {  createParser,  ParsedEvent,  ReconnectInterval,} from "eventsource-parser";export interface OpenAIStreamPayload { model: string; // this is a list of messages to give ChatGPT messages: { role: "user"; content: string }[]; stream: boolean;}  export async function OpenAIStream(payload: OpenAIStreamPayload) { const encoder = new TextEncoder(); const decoder = new TextDecoder();  let counter = 0; const res = await fetch("https://api.openai.com/v1/chat/completions", {  headers: {   "Content-Type": "application/json",   "Authorization": `Bearer ${process.env.OPENAI_API_KEY ?? ""}`,  },  method: "POST",  body: JSON.stringify(payload), });  const stream = new ReadableStream({  async start(controller) {   // callback   function onParse(event: ParsedEvent | ReconnectInterval) {    if (event.type === "event") {     const data = event.data;     // https://beta.openai.com/docs/api-reference/completions/create#completions/create-stream     if (data === "[DONE]") {      controller.close();      return;     }     try {      const json = JSON.parse(data);      // get the text response from ChatGPT      const text = json.choices[0]?.delta?.content;      if (!text) return;      if (counter < 2 && (text.match(/\n/) || []).length) {       // this is a prefix character (i.e., "\n\n"), do nothing       return;      }      const queue = encoder.encode(text);      controller.enqueue(queue);      counter++;     } catch (e) {      // maybe parse error      controller.error(e);     }    }   }     // stream response (SSE) from OpenAI may be fragmented into multiple chunks   // this ensures we properly read chunks and invoke an event for each SSE event stream   const parser = createParser(onParse);   // https://web.dev/streams/#asynchronous-iteration   for await (const chunk of res.body as any) {    parser.feed(decoder.decode(chunk));   }  }, });     return stream;}

// pages/api/qa.tsimport { OpenAIStream, OpenAIStreamPayload } from "../../utils/OpenAIStream";export const config = {  // We are using Vercel edge function for this endpoint  runtime: "edge",};interface RequestPayload { prompt: string;}const handler = async (req: Request, res: Response): Promise<Response> => { const { prompt } = (await req.json()) as RequestPayload; if (!prompt) {  return new Response("No prompt in the request", { status: 400 }); } const payload: OpenAIStreamPayload = {  model: "gpt-3.5-turbo",  messages: [{ role: "user", content: prompt }],  stream: true, }; const stream = await OpenAIStream(payload); return new Response(stream);};export default handler;

// theme.config.tsxconst Search = () => {  const [open, setOpen] = useState(false);  const [question, setQuestion] = useState("");  const [answer, setAnswer] = useState("");  const answerQuestion = async (e: any) => {    e.preventDefault();    setAnswer("");    // build the contextualized prompt    const promptResponse = await fetch("/api/buildPrompt", {      method: "POST",      headers: {        "Content-Type": "application/json",      },      body: JSON.stringify({        prompt: question,      }),    });    const promptData = await promptResponse.json();    // send it to ChatGPT    const response = await fetch("/api/qa", {      method: "POST",      headers: {        "Content-Type": "application/json",      },      body: JSON.stringify({        prompt: promptData.prompt,      }),    });    if (!response.ok) {      throw new Error(response.statusText);    }    const data = response.body;    if (!data) {      return;    }        const reader = data.getReader();    const decoder = new TextDecoder();    let done = false;    // read the streaming ChatGPT answer    while (!done) {      const { value, done: doneReading } = await reader.read();      done = doneReading;      const chunkValue = decoder.decode(value);      // update our interface with the answer      setAnswer((prev) => prev + chunkValue);    }  };  return (    <>      <input        placeholder="Ask a question"        onClick={() => setOpen(true)}        type="text"      />      <Modal open={open} onClose={() => setOpen(false)}>        <form onSubmit={answerQuestion} className="nx-flex nx-gap-3">          <input            placeholder="Ask a question"            type="text"            value={question}            onChange={(e) => setQuestion(e.target.value)}          />          <button type="submit">            Ask          </button>        </form>        <p>          {answer}        </p>      </Modal>    </>  );}

# .github/workflows/index.yamlname: Index documentationon:  push:    branches:      - mainjobs:  index:    runs-on: ubuntu-latest    steps:      - uses: actions/checkout@v2      - uses: actions/setup-node@v2        with:    node-version: 14      - run: npm install      - run: node scripts/sync.js  env:    EMBEDBASE_API_KEY: ${{ secrets.EMBEDBASE_API_KEY }}