Merge pull request #67 from BohuTANG/dev-source

feat: add SOURCES

app/src/configs/config.rs

@@ -12,8 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::fmt::{Debug, Formatter};
-use std::{env, fmt};
+use std::env;
+use std::fmt;
+use std::fmt::Debug;
+use std::fmt::Formatter;
 
 use anyhow::Result;
 use clap::Parser;

app/src/dal/db.rs

@@ -13,15 +13,17 @@
 // limitations under the License.
 
 use anyhow::Result;
-use chrono::{DateTime, Utc};
+use chrono::DateTime;
+use chrono::Utc;
 use databend_driver::DatabendConnection;
 use log::info;
 use tokio::time::Instant;
 use tokio_stream::StreamExt;
 
 use crate::base::escape_sql_string;
+use crate::remove_markdown_links;
+use crate::Config;
 use crate::SnippetFiles;
-use crate::{remove_markdown_links, Config};
 
 // Max completion tokens.
 const MAX_COMPLETION_LENGHT: usize = 8000;
@@ -142,21 +144,24 @@ impl DatabendDriver {
         let mut similar_distances = vec![];
 
         let sql = format!(
-            "SELECT content, distance FROM (SELECT content, cosine_distance({}, embedding) AS distance FROM {}.{} WHERE length(embedding) > 0 AND length(content)>{} ORDER BY distance ASC LIMIT {}) WHERE distance <={}",
+            "SELECT path, content, cosine_distance({}, embedding) AS distance FROM {}.{} WHERE length(embedding) > 0 AND length(content)>{} AND distance <= {} ORDER BY distance ASC LIMIT {}",
             query_embedding,
             self.database,
             self.table,
             self.min_content_length,
+            self.min_distance,
             self.top,
-            self.min_distance
         );
 
-        type RowResult = (String, f32);
+        type RowResult = (String, String, f32);
         let mut rows = self.conn.query_iter(&sql).await?;
         while let Some(row) = rows.next().await {
             let section_tuple: RowResult = row?.try_into()?;
-            similar_sections.push(section_tuple.0);
-            similar_distances.push(section_tuple.1);
+            similar_sections.push(format!(
+                "\nContent: {}\nSource: {}",
+                section_tuple.1, section_tuple.0
+            ));
+            similar_distances.push(section_tuple.2);
         }
         Ok((similar_sections, similar_distances))
     }

app/src/files/file.rs

@@ -16,7 +16,8 @@ use std::fs;
 use std::time::SystemTime;
 
 use anyhow::Result;
-use walkdir::{DirEntry, WalkDir};
+use walkdir::DirEntry;
+use walkdir::WalkDir;
 
 #[derive(Clone, Debug)]
 pub struct FileMeta {

app/src/files/markdown.rs

@@ -21,9 +21,10 @@ use comrak::parse_document;
 use comrak::Arena;
 use comrak::ComrakOptions;
 
+use crate::replace_multiple_spaces;
 use crate::Parse;
+use crate::SnippetFile;
 use crate::SnippetFiles;
-use crate::{replace_multiple_spaces, SnippetFile};
 
 pub struct Markdown;
 

app/tests/it/base/string.rs

@@ -12,7 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use askbend::{remove_markdown_links, replace_multiple_spaces, LengthWithoutSymbols};
+use askbend::remove_markdown_links;
+use askbend::replace_multiple_spaces;
+use askbend::LengthWithoutSymbols;
 
 #[test]
 fn test_remove_markdown_links() {

app/tests/it/files/markdown.rs

@@ -28,7 +28,7 @@ pub fn test_markdown_files() -> Result<()> {
 
     let markdowns = Markdown::parse_multiple(&files)?;
     for markdown in &markdowns.snippet_files {
-        //assert_eq!(markdown.file_path, "tests/testdata/hash.md");
+        // assert_eq!(markdown.file_path, "tests/testdata/hash.md");
         for section in &markdown.code_snippets {
             println!(
                 "*****file:{}, ++++++++++--{:?}",

conf/askbend.toml

@@ -26,13 +26,10 @@ top = 3
 product = "Databend"
 prompt = '''
 You are a very enthusiastic {{product}} AI who loves to help people! Given the following information from the {{product}} documentation, answer the user's question using only that information, outputted in markdown format.
+Create a final answer with references ("SOURCES").
 
 If you are unsure and the answer is not explicitly written in the documentation, say "Sorry, I don't know how to help with that."
-
-Always include related code snippets if available.
-Here is the {{product}} documentation: {{context}}
-
-Answer my next question using only the above documentation, emember to adhere to the following guidelines while answering:
+Please follow these instructions:
 - Refrain from creating answers not found in the documentation.
 - If the answer is not explicitly mentioned in the documentation, respond with "Sorry, I don't know how to help with that."
 - Present your answer in markdown format, including code snippets if have, format the code snippets with SQL type if necessary.
@@ -41,4 +38,10 @@ Answer my next question using only the above documentation, emember to adhere to
 - Do not change the SQL syntax, please don't make up the function.
 - Do not change explain any code snippets.
 - Make the whole answer as short as possible to keep the code snippets.
-Here is my question: {{query}}'''
+
+=========
+{{context}}
+=========
+QUESTION: {{query}}
+FINAL ANSWER:
+'''

rust-toolchain.toml

@@ -1,3 +1,3 @@
 [toolchain]
-channel = "stable"
+channel = "nightly-2023-03-10"
 components = ["rustfmt", "clippy", "rust-src"]