12/8/25, 8:45 AM src

Folder src
25 printable files

src\api\[Link]
src\api\file_upload.rs
src\api\[Link]
src\api\[Link]
src\core\[Link]
src\core\[Link]
src\core\[Link]
src\core\[Link]
src\core\sentence_pipeline.rs
src\core\[Link]
src\core\[Link]
src\core\[Link]
src\core\[Link]
src\extraction\[Link]
src\extraction\[Link]
src\extraction\[Link]
src\extraction\[Link]
src\[Link]
src\[Link]
src\models\[Link]
src\models\[Link]
src\models\[Link]
src\models\[Link]
src\models\sentence_analysis.rs
src\sentence\[Link]

src\api\[Link]

1 //! Application error types

2
3 use axum::{
4 http::StatusCode,
5 response::{IntoResponse, Response},
6 Json,
7 };
8 use serde::Serialize;
9 use thiserror::Error;
10
11 /// Application error types
12 #[derive(Debug, Error)]
13 pub enum AppError {
14 #[error("Too many documents: {0}, maximum allowed is 100")]
15 TooManyDocuments(usize),
16
17 #[error("Empty document at index {0}")]
18 EmptyDocument(usize),
19
20 #[error("No documents provided")]
21 NoDocuments,
22
23 #[error("Not enough documents: minimum 2 required for comparison, got {0}")]
24 NotEnoughDocuments(usize),
25
26 #[error("Document at index {0} exceeds maximum length of {1} characters")]
27 DocumentTooLong(usize, usize),

localhost:33055/06e81d38-77a8-40c6-bafe-ec10aec40970/ 1/39
12/8/25, 8:45 AM src

28
29 #[error("Internal server error: {0}")]
30 Internal(#[from] anyhow::Error),
31 }
32
33 /// Error response body
34 #[derive(Debug, Serialize)]
35 struct ErrorResponse {
36 error: String,
37 code: String,
38 }
39
40 impl IntoResponse for AppError {
41 fn into_response(self) -> Response {
42 let (status, code) = match &self {
43 AppError::TooManyDocuments(_) => (StatusCode::BAD_REQUEST,
"TOO_MANY_DOCUMENTS"),
44 AppError::EmptyDocument(_) => (StatusCode::BAD_REQUEST, "EMPTY_DOCUMENT"),
45 AppError::NoDocuments => (StatusCode::BAD_REQUEST, "NO_DOCUMENTS"),
46 AppError::NotEnoughDocuments(_) => (StatusCode::BAD_REQUEST,
"NOT_ENOUGH_DOCUMENTS­
"),
47 AppError::DocumentTooLong(_, _) => (StatusCode::BAD_REQUEST,
"DOCUMENT_TOO_LONG"),
48 AppError::Internal(_) => (StatusCode::INTERNAL_SERVER_ERRO­
R, "INTERNAL_ERROR"),
49 };
50
51 let body = ErrorResponse {
52 error: self.to_string(),
53 code: code.to_string(),
54 };
55
56 (status, Json(body)).into_response()
57 }
58 }
59

localhost:33055/06e81d38-77a8-40c6-bafe-ec10aec40970/ 2/39
12/8/25, 8:45 AM src

src\api\file_upload.rs

1 //! File upload handler for sentence-level analysis

2
3 use axum::extract::Multipart;
4 use axum::http::StatusCode;
5 use axum::response::{IntoResponse, Response};
6 use axum::Json;
7 use std::time::Instant;
8
9 use crate::extraction::{extract_text, FileType};
10 use crate::sentence::split_sentences;
11 use crate::core::{analyze_sentence_sim­
ilarity, SentenceDocument};
12 use crate::models::{SentenceAnalysisResp­
onse, AnalysisMetadata};
13
14 /// Constants for file upload limits
15 const MAX_FILE_SIZE: usize = 10 * 1024 * 1024; // 10 MB
16 const MAX_TOTAL_SIZE: usize = 50 * 1024 * 1024; // 50 MB
17 const MAX_FILES: usize = 5;
18 const MIN_FILES: usize = 2;
19 const DEFAULT_THRESHOLD: f32 = 0.70;
20
21 /// Handler for POST /api/analyze with multipart file upload
22 ///
23 /// Accepts up to 5 files (PDF/DOCX/TXT) and returns sentence-level similarity analysis.
24 pub async fn analyze_files_handle­
r(
25 mut multipart: Multipart,
26 ) -> Result<Json<SentenceAnalysisResp­
onse>, FileUploadError> {
27 let start_time = Instant::now();
28
29 // Collect files and threshold from multipart form
30 let (files, threshold) = extract_files_and_th­
reshold(&mut multipart).await?;
31
32 // Validate minimum files
33 if [Link]() < MIN_FILES {
34 return Err(FileUploadError::NotEnoughFiles(MIN_FILES));
35 }
36
37 // Extract text from files
38 let documents: Result<Vec<SentenceDocument>, FileUploadError> = files
39 .into_iter()
40 .map(|(filename, data)| {
41 // Detect file type
42 let file_type = FileType::from_filename(&filename)
43 .ok_or_else(|| FileUploadError::UnsupportedFileType([Link]()))?;
44
45 // Extract text
46 let text = extract_text(&data, file_type)
47 .map_err(|e| FileUploadError::ExtractionError([Link](), e))?;
48
49 // Split into sentences
50 let sentences = split_sentences(&text);
51

localhost:33055/06e81d38-77a8-40c6-bafe-ec10aec40970/ 3/39
12/8/25, 8:45 AM src

52 if sentences.is_empty() {
53 return Err(FileUploadError::EmptyDocument(filename));
54 }
55
56 Ok(SentenceDocument::new(filename, sentences))
57 })
58 .collect();
59
60 let documents = documents?;
61
62 // Count total sentences
63 let total_sentences: usize = [Link]().map(|d| [Link]()).sum();
64
65 // Analyze similarity
66 let (matches, global_similarity) = analyze_sentence_sim­
ilarity(&documents, threshold);
67
68 // Compute processing time
69 let processing_time_ms = start_time.elapsed().as_millis() as u64;
70
71 // Build metadata
72 let metadata = AnalysisMetadata::new(
73 [Link](),
74 total_sentences,
75 processing_time_ms,
76 threshold,
77 );
78
79 // Build response
80 let response = SentenceAnalysisResp­
onse::new(metadata, matches, global_similarity);
81
82 Ok(Json(response))
83 }
84
85 /// Health check endpoint
86 pub async fn health_handler() -> &'static str {
87 "OK"
88 }
89
90 /// Errors that can occur during file upload and processing
91 #[derive(Debug)]
92 pub enum FileUploadError {
93 InvalidMultipart(String),
94 MissingFilename,
95 ReadError(String),
96 FileTooLarge(String, usize),
97 TotalSizeTooLarge(usize),
98 TooManyFiles(usize),
99 NotEnoughFiles(usize),
100 UnsupportedFileType(String),
101 ExtractionError(String, String),
102 EmptyDocument(String),
103 InvalidThreshold(String),
104 InvalidThresholdRang­
e(f32),
105 }
localhost:33055/06e81d38-77a8-40c6-bafe-ec10aec40970/ 4/39
12/8/25, 8:45 AM src

106
107 impl IntoResponse for FileUploadError {
108 fn into_response(self) -> Response {
109 let (status, message) = match self {
110 FileUploadError::InvalidMultipart(e) => {
111 (StatusCode::BAD_REQUEST, format!("Invalid multipart data: {}", e))
112 }
113 FileUploadError::MissingFilename => {
114 (StatusCode::BAD_REQUEST, "File is missing filename".to_string())
115 }
116 FileUploadError::ReadError(e) => {
117 (StatusCode::BAD_REQUEST, format!("Error reading file: {}", e))
118 }
119 FileUploadError::FileTooLarge(filename, max) => {
120 (
121 StatusCode::PAYLOAD_TOO_LARGE,
122 format!("File '{}' exceeds maximum size of {} bytes", filename, max),
123 )
124 }
125 FileUploadError::TotalSizeTooLarge(max) => {
126 (
127 StatusCode::PAYLOAD_TOO_LARGE,
128 format!("Total upload size exceeds maximum of {} bytes", max),
129 )
130 }
131 FileUploadError::TooManyFiles(max) => {
132 (
133 StatusCode::BAD_REQUEST,
134 format!("Too many files. Maximum allowed: {}", max),
135 )
136 }
137 FileUploadError::NotEnoughFiles(min) => {
138 (
139 StatusCode::BAD_REQUEST,
140 format!("Not enough files. Minimum required: {}", min),
141 )
142 }
143 FileUploadError::UnsupportedFileType(filename) => {
144 (
145 StatusCode::BAD_REQUEST,
146 format!("Unsupported file type: {}. Allowed: PDF, DOCX, TXT",
filename),
147 )
148 }
149 FileUploadError::ExtractionError(filename, error) => {
150 (
151 StatusCode::UNPROCESSABLE_ENTITY­
,
152 format!("Failed to extract text from '{}': {}", filename, error),
153 )
154 }
155 FileUploadError::EmptyDocument(filename) => {
156 (
157 StatusCode::BAD_REQUEST,
158 format!("Document '{}' contains no text or sentences", filename),

localhost:33055/06e81d38-77a8-40c6-bafe-ec10aec40970/ 5/39
12/8/25, 8:45 AM src

159 )
160 }
161 FileUploadError::InvalidThreshold(value) => {
162 (
163 StatusCode::BAD_REQUEST,
164 format!("Invalid threshold value: '{}'. Must be a number between 0.0
and 1.0", value),
165 )
166 }
167 FileUploadError::InvalidThresholdRang­
e(value) => {
168 (
169 StatusCode::BAD_REQUEST,
170 format!("Threshold {} out of range. Must be between 0.0 and 1.0",
value),
171 )
172 }
173 };
174
175 (status, message).into_response()
176 }
177 }
178
179 /// Extract files and threshold from multipart form data
180 async fn extract_files_and_th­
reshold(
181 multipart: &mut Multipart,
182 ) -> Result<(Vec<(String, Vec<u8>)>, f32), FileUploadError> {
183 let mut files: Vec<(String, Vec<u8>)> = Vec::new();
184 let mut threshold_value: Option<f32> = None;
185 let mut total_size = 0usize;
186
187 while let Some(field) = multipart.next_field().await
188 .map_err(|e| FileUploadError::InvalidMultipart(e.to_string()))? {
189
190 let field_name = [Link]().unwrap_or("").to_string();
191
192 // Check if this is the threshold field
193 if field_name == "threshold" {
194 let threshold_str = [Link]().await
195 .map_err(|e| FileUploadError::ReadError(e.to_string()))?;
196
197 threshold_value = Some(
198 threshold_str.trim().parse::<f32>()
199 .map_err(|_| FileUploadError::InvalidThreshold(threshold_str))?
200 );
201 continue;
202 }
203
204 // Otherwise, it's a file field
205 let filename = field.file_name()
206 .ok_or(FileUploadError::MissingFilename)?
207 .to_string();
208
209 let data = [Link]().await
210 .map_err(|e| FileUploadError::ReadError(e.to_string()))?

localhost:33055/06e81d38-77a8-40c6-bafe-ec10aec40970/ 6/39
12/8/25, 8:45 AM src

211 .to_vec();
212
213 // Check individual file size
214 if [Link]() > MAX_FILE_SIZE {
215 return Err(FileUploadError::FileTooLarge(filename, MAX_FILE_SIZE));
216 }
217
218 total_size += [Link]();
219
220 // Check total size
221 if total_size > MAX_TOTAL_SIZE {
222 return Err(FileUploadError::TotalSizeTooLarge(MAX_TOTAL_SIZE));
223 }
224
225 [Link]((filename, data));
226
227 // Check max files
228 if [Link]() > MAX_FILES {
229 return Err(FileUploadError::TooManyFiles(MAX_FILES));
230 }
231 }
232
233 // Use provided threshold or default
234 let threshold = threshold_value.unwrap_or(DEFAULT_THRESHOLD);
235
236 // Validate threshold range
237 if threshold < 0.0 || threshold > 1.0 {
238 return Err(FileUploadError::InvalidThresholdRang­
e(threshold));
239 }
240
241 Ok((files, threshold))
242 }
243

localhost:33055/06e81d38-77a8-40c6-bafe-ec10aec40970/ 7/39
12/8/25, 8:45 AM src

src\api\[Link]

1 //! HTTP API module

2
3 mod error;
4 mod server;
5 mod file_upload;
6
7 pub use error::AppError;
8 pub use file_upload::{analyze_files_handle­
r, health_handler};
9 pub use server::{create_router, run_server};
10

localhost:33055/06e81d38-77a8-40c6-bafe-ec10aec40970/ 8/39
12/8/25, 8:45 AM src

src\api\[Link]

1 //! HTTP server configuration

2
3 use axum::{
4 routing::{get, post},
5 Router,
6 };
7 use std::net::SocketAddr;
8 use tower_http::cors::{Any, CorsLayer};
9 use tracing::info;
10
11 use super::file_upload::{analyze_files_handle­
r, health_handler};
12
13 /// Creates the Axum router with all routes configured
14 pub fn create_router() -> Router {
15 // Configure CORS
16 let cors = CorsLayer::new()
17 .allow_origin(Any)
18 .allow_methods(Any)
19 .allow_headers(Any);
20
21 Router::new()
22 .route("/health", get(health_handler))
23 .route("/api/analyze", post(analyze_files_handle­
r))
24 .layer(cors)
25 }
26
27 /// Runs the HTTP server
28 pub async fn run_server(port: u16) -> anyhow::Result<()> {
29 let app = create_router();
30 let addr = SocketAddr::from(([0, 0, 0, 0], port));
31
32 info!("🚀 Server starting on [Link] addr);
33 info!("📊 POST /api/analyze - Analyze sentence-level similarity (multipart file
upload)");
34 info!("❤️ GET /health - Health check");
35
36 let listener = tokio::net::TcpListener::bind(addr).await?;
37 axum::serve(listener, app).await?;
38
39 Ok(())
40 }
41
42

localhost:33055/06e81d38-77a8-40c6-bafe-ec10aec40970/ 9/39
12/8/25, 8:45 AM src

src\core\[Link]

1 //! Inverse Document Frequency calculation - pure function

2
3 use std::collections::HashMap;
4
5 /// Computes Inverse Document Frequency (IDF) across all documents.
6 /// Uses smoothed IDF: IDF = log((N + 1) / (df + 1)) + 1
7 /// This prevents division by zero and ensures non-zero IDF values.
8 ///
9 /// # Arguments
10 /// * `tfs` - Slice of Term Frequency maps, one per document
11 ///
12 /// # Returns
13 /// A HashMap mapping each term to its IDF value
14 ///
15 /// # Example
16 /// ```
17 /// use document_similarity_­
analyzer::core::compute_idf;
18 /// use std::collections::HashMap;
19 ///
20 /// let tf1: HashMap<String, f32> = [("hello".to_string(), 0.5)].into_iter().collect();
21 /// let tf2: HashMap<String, f32> = [("world".to_string(), 0.5)].into_iter().collect();
22 /// let idf = compute_idf(&[tf1, tf2]);
23 /// ```
24 pub fn compute_idf(tfs: &[HashMap<String, f32>]) -> HashMap<String, f32> {
25 if tfs.is_empty() {
26 return HashMap::new();
27 }
28
29 let n = [Link]() as f32;
30
31 // Collect all unique terms from all documents and count document frequency
32 let document_frequency = tfs
33 .iter()
34 .flat_map(|tf| [Link]())
35 .fold(HashMap::new(), |mut acc, term| {
36 *[Link]([Link]()).or_insert(0) += 1;
37 acc
38 });
39
40 // Calculate smoothed IDF for each term
41 // Using: IDF = log((N + 1) / (df + 1)) + 1
42 // This ensures IDF is always positive and handles edge cases
43 document_frequency
44 .into_iter()
45 .map(|(term, df)| {
46 let idf = ((n + 1.0) / (df as f32 + 1.0)).ln() + 1.0;
47 (term, idf)
48 })
49 .collect()
50 }
51

localhost:33055/06e81d38-77a8-40c6-bafe-ec10aec40970/ 10/39
12/8/25, 8:45 AM src

src\core\[Link]

1 //! Similarity Matrix generation - parallel computation

2
3 use super::cosine_similarity;
4 use rayon::prelude::*;
5
6 pub fn compute_similarity_m­
atrix(vectors: &[Vec<f32>]) -> Vec<Vec<f32>> {
7 let n = [Link](); //TODO penamaan variabel
8
9 if n == 0 {
10 return vec![];
11 }
12
13 // Parallel computation of similarity matrix
14 (0..n)
15 .into_par_iter()
16 .map(|i| {
17 (0..n)
18 .map(|j| {
19 if i == j {
20 1.0 // Diagonal is always 1.0
21 } else if i < j {
22 // Compute similarity for upper triangle
23 cosine_similarity(&vectors[i], &vectors[j])
24 } else {
25 // For lower triangle, we'll compute it too
26 // (could optimize by computing upper only and copying)
27 cosine_similarity(&vectors[i], &vectors[j])
28 }
29 })
30 .collect()
31 })
32 .collect()
33 }
34

localhost:33055/06e81d38-77a8-40c6-bafe-ec10aec40970/ 11/39
12/8/25, 8:45 AM src

src\core\[Link]

1 //! Core processing functions - all pure functions with no side effects
2
3 mod normalize;
4 mod tokenize;
5 mod tf;
6 mod idf;
7 mod vectorize;
8 mod similarity;
9 mod matrix;
10 mod sentence_pipeline;
11
12 pub use normalize::normalize_text;
13 pub use tokenize::tokenize;
14 pub use tf::compute_tf;
15 pub use idf::compute_idf;
16 pub use vectorize::{vectorize, compute_tfidf_vector­
};
17 pub use similarity::{cosine_similarity, compute_cosine_simil­
arity};
18 pub use matrix::compute_similarity_m­
atrix;
19 pub use sentence_pipeline::{analyze_sentence_sim­
ilarity, SentenceDocument};
20

localhost:33055/06e81d38-77a8-40c6-bafe-ec10aec40970/ 12/39
12/8/25, 8:45 AM src

src\core\[Link]

1 //! Text normalization - pure function

2
3 /// Normalizes text by converting to lowercase, removing punctuation,
4 /// and collapsing multiple whitespace into single space.
5
6 pub fn normalize_text(text: &str) -> String {
7 [Link]()
8 .map(|c| {
9 if c.is_ascii_punctuation­
() {
10 ' '
11 } else {
12 c.to_ascii_lowercase()
13 }
14 })
15 .collect::<String>()
16 .split_whitespace()
17 .collect::<Vec<&str>>()
18 .join(" ")
19 }
20

localhost:33055/06e81d38-77a8-40c6-bafe-ec10aec40970/ 13/39
12/8/25, 8:45 AM src

src\core\sentence_pipeline.rs

1 //! Sentence-level document similarity analysis pipeline

2
3 use std::collections::HashMap;
4 use rayon::prelude::*;
5
6 use crate::core::{compute_tf, compute_idf, normalize_text, tokenize, compute_tfidf_vector­
,
compute_cosine_simil­
arity};
7 use crate::models::{SentenceMatch, GlobalSimilarity};
8
9 /// Represents a document with its sentences
10 #[derive(Debug, Clone)]
11 pub struct SentenceDocument {
12 pub filename: String,
13 pub sentences: Vec<String>,
14 }
15
16 impl SentenceDocument {
17 pub fn new(filename: String, sentences: Vec<String>) -> Self {
18 Self { filename, sentences }
19 }
20 }
21
22 /// Represents a sentence with its TF-IDF vector
23 #[derive(Debug, Clone)]
24 struct SentenceVector {
25 doc_index: usize,
26 sentence_index: usize,
27 vector: HashMap<String, f32>,
28 }
29
30 /// Analyze sentence-level similarity across multiple documents
31 pub fn analyze_sentence_sim­
ilarity(
32 documents: &[SentenceDocument],
33 threshold: f32,
34 ) -> (Vec<SentenceMatch>, Vec<GlobalSimilarity>) {
35 // Step 1: Flatten all sentences with their document context
36 let all_sentences: Vec<(usize, usize, String)> = documents
37 .iter()
38 .enumerate()
39 .flat_map(|(doc_idx, doc)| {
40 [Link]
41 .iter()
42 .enumerate()
43 .map(move |(sent_idx, sentence)| (doc_idx, sent_idx, [Link]()))
44 })
45 .collect();
46
47 if all_sentences.is_empty() {
48 return (vec![], vec![]);
49 }
50
51 // Step 2: Process each sentence (normalize + tokenize)
localhost:33055/06e81d38-77a8-40c6-bafe-ec10aec40970/ 14/39
12/8/25, 8:45 AM src

52 let processed_sentences: Vec<(usize, usize, String, Vec<String>)> = all_sentences

53 .par_iter()
54 .map(|(doc_idx, sent_idx, text)| {
55 let normalized = normalize_text(text);
56 let tokens = tokenize(&normalized);
57 (*doc_idx, *sent_idx, [Link](), tokens)
58 })
59 .collect();
60
61 // Step 3: Compute TF for each sentence
62 let sentence_tfs: Vec<(usize, usize, String, HashMap<String, f32>)> =
processed_sentences
63 .into_par_iter()
64 .map(|(doc_idx, sent_idx, text, tokens)| {
65 let tf = compute_tf(&tokens);
66 (doc_idx, sent_idx, text, tf)
67 })
68 .collect();
69
70 // Step 4: Compute global IDF from all sentences
71 let tfs_only: Vec<HashMap<String, f32>> = sentence_tfs
72 .iter()
73 .map(|(_, _, _, tf)| [Link]())
74 .collect();
75 let global_idf = compute_idf(&tfs_only);
76
77 // Step 5: Compute TF-IDF vectors for each sentence
78 let sentence_vectors: Vec<SentenceVector> = sentence_tfs
79 .into_par_iter()
80 .map(|(doc_idx, sent_idx, _text, tf)| {
81 let vector = compute_tfidf_vector­
(&tf, &global_idf);
82 SentenceVector {
83 doc_index: doc_idx,
84 sentence_index: sent_idx,
85 vector,
86 }
87 })
88 .collect();
89
90 // Step 6: Compute pairwise similarities (cross-document only)
91 let matches = compute_sentence_mat­
ches(&sentence_vectors, documents, threshold);
92
93 // Step 7: Compute global document similarities
94 let global_similarities = compute_global_simil­
arities(&sentence_vectors, documents);
95
96 (matches, global_similarities)
97 }
98
99 fn compute_sentence_mat­
ches(
100 vectors: &[SentenceVector],
101 documents: &[SentenceDocument],
102 threshold: f32,
103 ) -> Vec<SentenceMatch> {
104 // Generate all pairs, filter by threshold, and sort by similarity descending

localhost:33055/06e81d38-77a8-40c6-bafe-ec10aec40970/ 15/39
12/8/25, 8:45 AM src

105 let mut matches: Vec<SentenceMatch> = vectors

106 .iter()
107 .enumerate()
108 .flat_map(|(i, vec_a)| {
109 [Link]().skip(i + 1).filter_map(move |vec_b| {
110 // Only compare sentences from different documents
111 if vec_a.doc_index == vec_b.doc_index {
112 return None;
113 }
114
115 let similarity = compute_cosine_simil­
arity(&vec_a.vector, &vec_b.vector);
116
117 if similarity >= threshold {
118 let source_doc = documents[vec_a.doc_index].[Link]();
119 let target_doc = documents[vec_b.doc_index].[Link]();
120
121 // Get actual sentence text
122 let source_sentence =
documents[vec_a.doc_index].sentences[vec_a.sentence_index].clone();
123 let target_sentence =
documents[vec_b.doc_index].sentences[vec_b.sentence_index].clone();
124
125 Some(SentenceMatch::new(
126 source_doc,
127 vec_a.sentence_index,
128 source_sentence,
129 target_doc,
130 vec_b.sentence_index,
131 target_sentence,
132 similarity,
133 ))
134 } else {
135 None
136 }
137 })
138 })
139 .collect();
140
141 // Sort by similarity descending (must use mut here as sort_by requires &mut self)
142 matches.sort_by(|a, b| [Link].partial_cmp(&[Link]).unwrap());
143
144 matches
145 }
146
147 /// Compute global similarity between document pairs
148 fn compute_global_simil­
arities(
149 vectors: &[SentenceVector],
150 documents: &[SentenceDocument],
151 ) -> Vec<GlobalSimilarity> {
152 // Group vectors by document using fold (more functional than mut + for loop)
153 let doc_vectors: HashMap<usize, Vec<&SentenceVector>> = vectors
154 .iter()
155 .fold(HashMap::new(), |mut acc, vector| {
156 [Link](vector.doc_index)

localhost:33055/06e81d38-77a8-40c6-bafe-ec10aec40970/ 16/39
12/8/25, 8:45 AM src

157 .or_insert_with(Vec::new)
158 .push(vector);
159 acc
160 });
161
162 let empty_vec: Vec<&SentenceVector> = Vec::new();
163
164 // Collect all document pairs first, then map to compute similarities
165 let doc_pairs: Vec<(usize, usize)> = (0..[Link]())
166 .flat_map(|doc_a_idx| {
167 ((doc_a_idx + 1)..[Link]()).map(move |doc_b_idx| (doc_a_idx,
doc_b_idx))
168 })
169 .collect();
170
171 // Compute similarity for each pair
172 let mut global_sims: Vec<GlobalSimilarity> = doc_pairs
173 .iter()
174 .filter_map(|(doc_a_idx, doc_b_idx)| {
175 let vecs_a = doc_vectors.get(doc_a_idx).unwrap_or(&empty_vec);
176 let vecs_b = doc_vectors.get(doc_b_idx).unwrap_or(&empty_vec);
177
178 if vecs_a.is_empty() || vecs_b.is_empty() {
179 return None;
180 }
181
182 // Compute all cross-document sentence similarities using flat_map
183 let similarities: Vec<f32> = vecs_a
184 .iter()
185 .flat_map(|vec_a| {
186 vecs_b.iter().map(|vec_b| {
187 compute_cosine_simil­
arity(&vec_a.vector, &vec_b.vector)
188 })
189 })
190 .collect();
191
192 // Average similarity
193 let avg_similarity = [Link]().sum::<f32>() / [Link]() as
f32;
194
195 Some(GlobalSimilarity::new(
196 documents[*doc_a_idx].[Link](),
197 documents[*doc_b_idx].[Link](),
198 avg_similarity,
199 ))
200 })
201 .collect();
202
203 // Sort by score descending (must use mut here as sort_by requires &mut self)
204 global_sims.sort_by(|a, b| [Link].partial_cmp(&[Link]).unwrap());
205
206 global_sims
207 }
208

localhost:33055/06e81d38-77a8-40c6-bafe-ec10aec40970/ 17/39
12/8/25, 8:45 AM src

src\core\[Link]

1 //! Cosine Similarity calculation - pure function

2
3 use std::collections::HashMap;
4
5 /// Computes cosine similarity between two vectors.
6 /// Formula: (A · B) / (||A|| * ||B||)
7 pub fn cosine_similarity(vec_a: &[f32], vec_b: &[f32]) -> f32 {
8 if vec_a.len() != vec_b.len() || vec_a.is_empty() {
9 return 0.0;
10 }
11
12 // Compute dot product: A · B
13 let dot_product: f32 = vec_a
14 .iter()
15 .zip(vec_b.iter())
16 .map(|(a, b)| a * b)
17 .sum();
18
19 // Compute magnitudes: ||A|| and ||B||
20 let magnitude_a: f32 = vec_a.iter().map(|x| x * x).sum::<f32>().sqrt();
21 let magnitude_b: f32 = vec_b.iter().map(|x| x * x).sum::<f32>().sqrt();
22
23 // Handle zero magnitude case
24 if magnitude_a == 0.0 || magnitude_b == 0.0 {
25 return 0.0;
26 }
27
28 dot_product / (magnitude_a * magnitude_b)
29 }
30
31 /// Compute cosine similarity between two HashMap-based TF-IDF vectors
32 ///
33 /// # Arguments
34 /// * `vec_a` - First vector as HashMap
35 /// * `vec_b` - Second vector as HashMap
36 ///
37 /// # Returns
38 /// Cosine similarity value between -1.0 and 1.0
39 pub fn compute_cosine_simil­
arity(
40 vec_a: &HashMap<String, f32>,
41 vec_b: &HashMap<String, f32>,
42 ) -> f32 {
43 if vec_a.is_empty() || vec_b.is_empty() {
44 return 0.0;
45 }
46
47 // Compute dot product for common terms
48 let dot_product: f32 = vec_a
49 .iter()
50 .filter_map(|(term, a_val)| vec_b.get(term).map(|b_val| a_val * b_val))
51 .sum();

localhost:33055/06e81d38-77a8-40c6-bafe-ec10aec40970/ 18/39
12/8/25, 8:45 AM src

52
53 // Compute magnitudes
54 let magnitude_a: f32 = vec_a.values().map(|x| x * x).sum::<f32>().sqrt();
55 let magnitude_b: f32 = vec_b.values().map(|x| x * x).sum::<f32>().sqrt();
56
57 if magnitude_a == 0.0 || magnitude_b == 0.0 {
58 return 0.0;
59 }
60
61 dot_product / (magnitude_a * magnitude_b)
62 }
63

localhost:33055/06e81d38-77a8-40c6-bafe-ec10aec40970/ 19/39
12/8/25, 8:45 AM src

src\core\[Link]

1 //! Term Frequency calculation - pure function

2
3 use std::collections::HashMap;
4
5 /// Computes Term Frequency (TF) for a list of tokens.
6 /// TF = (number of times term appears) / (total number of terms)
7
8 pub fn compute_tf(tokens: &[String]) -> HashMap<String, f32> {
9 if tokens.is_empty() {
10 return HashMap::new();
11 }
12
13 let total = [Link]() as f32;
14
15 // Count occurrences using fold (functional approach)
16 let counts = [Link]().fold(HashMap::new(), |mut acc, token| {
17 *[Link]([Link]()).or_insert(0) += 1;
18 acc
19 });
20
21 counts
22 .into_iter()
23 .map(|(term, count)| (term, count as f32 / total))
24 .collect()
25 }
26

localhost:33055/06e81d38-77a8-40c6-bafe-ec10aec40970/ 20/39
12/8/25, 8:45 AM src

src\core\[Link]

1 //! Tokenization - pure function

2
3 /// Tokenizes text into a vector of words by splitting on whitespace.
4
5 pub fn tokenize(text: &str) -> Vec<String> {
6 text.split_whitespace()
7 .filter(|s| !s.is_empty())
8 .map(|s| s.to_string())
9 .collect()
10 }
11

localhost:33055/06e81d38-77a8-40c6-bafe-ec10aec40970/ 21/39
12/8/25, 8:45 AM src

src\core\[Link]

1 //! TF-IDF Vectorization - pure function

2
3 use std::collections::HashMap;
4
5 /// Converts TF and IDF into a TF-IDF vector based on vocabulary order.
6 pub fn vectorize(
7 tf: &HashMap<String, f32>,
8 idf: &HashMap<String, f32>,
9 vocabulary: &[String],
10 ) -> Vec<f32> {
11 vocabulary
12 .iter()
13 .map(|term| {
14 let tf_value = [Link](term).copied().unwrap_or(0.0);
15 let idf_value = [Link](term).copied().unwrap_or(0.0);
16 tf_value * idf_value
17 })
18 .collect()
19 }
20
21 /// Compute TF-IDF vector directly as HashMap (for sentence-level analysis)
22 pub fn compute_tfidf_vector­
(
23 tf: &HashMap<String, f32>,
24 idf: &HashMap<String, f32>,
25 ) -> HashMap<String, f32> {
26 [Link]()
27 .map(|(term, tf_value)| {
28 let idf_value = [Link](term).copied().unwrap_or(0.0);
29 ([Link](), tf_value * idf_value)
30 })
31 .collect()
32 }
33

localhost:33055/06e81d38-77a8-40c6-bafe-ec10aec40970/ 22/39
12/8/25, 8:45 AM src

src\extraction\[Link]

1 //! DOCX text extraction module

2
3 /// Extract text from DOCX file bytes
4 ///
5 /// Uses docx-rs library to parse DOCX (Open XML format) and extract text content.
6 /// Returns concatenated text from all paragraphs.
7 pub fn extract_docx(file_bytes: &[u8]) -> Result<String, String> {
8 docx_rs::read_docx(file_bytes)
9 .map_err(|e| format!("Failed to extract DOCX: {}", e))
10 .map(|docx| {
11 // Extract text from all paragraphs
12 let text_parts: Vec<String> = docx
13 .document
14 .children
15 .iter()
16 .filter_map(|child| {
17 match child {
18 docx_rs::DocumentChild::Paragraph(para) => {
19 let para_text: String = para
20 .children
21 .iter()
22 .filter_map(|p_child| {
23 match p_child {
24 docx_rs::ParagraphChild::Run(run) => {
25 let run_text: String = run
26 .children
27 .iter()
28 .filter_map(|r_child| {
29 match r_child {
30 docx_rs::RunChild::Text(text) => {
31 Some([Link]())
32 }
33 _ => None,
34 }
35 })
36 .collect::<Vec<_>>()
37 .join("");
38 if run_text.is_empty() {
39 None
40 } else {
41 Some(run_text)
42 }
43 }
44 _ => None,
45 }
46 })
47 .collect::<Vec<_>>()
48 .join(" ");
49
50 if para_text.trim().is_empty() {
51 None

localhost:33055/06e81d38-77a8-40c6-bafe-ec10aec40970/ 23/39
12/8/25, 8:45 AM src

52 } else {
53 Some(para_text)
54 }
55 }
56 _ => None,
57 }
58 })
59 .collect();
60
61 text_parts.join("\n")
62 })
63 }
64

localhost:33055/06e81d38-77a8-40c6-bafe-ec10aec40970/ 24/39
12/8/25, 8:45 AM src

src\extraction\[Link]

1 //! File extraction modules for PDF, DOCX, and TXT files
2
3 pub mod pdf;
4 pub mod docx;
5 pub mod txt;
6
7 pub use self::pdf::extract_pdf;
8 pub use self::docx::extract_docx;
9 pub use self::txt::extract_txt;
10
11 use std::path::Path;
12
13 /// Supported file types for extraction
14 #[derive(Debug, Clone, Copy, PartialEq)]
15 pub enum FileType {
16 Pdf,
17 Docx,
18 Txt,
19 }
20
21 impl FileType {
22 /// Detect file type from extension
23 pub fn from_extension(extension: &str) -> Option<Self> {
24 match extension.to_lowercase().as_str() {
25 "pdf" => Some(FileType::Pdf),
26 "docx" => Some(FileType::Docx),
27 "txt" => Some(FileType::Txt),
28 _ => None,
29 }
30 }
31
32 /// Detect file type from filename
33 pub fn from_filename(filename: &str) -> Option<Self> {
34 Path::new(filename)
35 .extension()
36 .and_then(|ext| ext.to_str())
37 .and_then(Self::from_extension)
38 }
39 }
40
41 /// Extract text from file bytes based on file type
42 pub fn extract_text(file_bytes: &[u8], file_type: FileType) -> Result<String, String> {
43 match file_type {
44 FileType::Pdf => extract_pdf(file_bytes),
45 FileType::Docx => extract_docx(file_bytes),
46 FileType::Txt => extract_txt(file_bytes),
47 }
48 }
49

localhost:33055/06e81d38-77a8-40c6-bafe-ec10aec40970/ 25/39
12/8/25, 8:45 AM src

src\extraction\[Link]

1 //! PDF text extraction module

2
3 /// Extract text from PDF file bytes
4 ///
5 /// Uses pdf-extract library to parse PDF and extract text content.
6 /// Returns concatenated text from all pages.
7 pub fn extract_pdf(file_bytes: &[u8]) -> Result<String, String> {
8 pdf_extract::extract_text_from_me­
m(file_bytes)
9 .map_err(|e| format!("Failed to extract PDF: {}", e))
10 .map(|text| [Link]().to_string())
11 }
12

localhost:33055/06e81d38-77a8-40c6-bafe-ec10aec40970/ 26/39
12/8/25, 8:45 AM src

src\extraction\[Link]

1 //! TXT text extraction module

2
3 /// Extract text from TXT file bytes
4 ///
5 /// Simply converts bytes to UTF-8 string.
6 /// Returns the decoded text content.
7 pub fn extract_txt(file_bytes: &[u8]) -> Result<String, String> {
8 String::from_utf8(file_bytes.to_vec())
9 .map_err(|e| format!("Failed to decode TXT as UTF-8: {}", e))
10 .map(|text| [Link]().to_string())
11 }
12

localhost:33055/06e81d38-77a8-40c6-bafe-ec10aec40970/ 27/39
12/8/25, 8:45 AM src

src\[Link]

1 //! Document Similarity Analyzer

2 //!
3 //! A backend service for analyzing document similarity using TF-IDF and Cosine Similarity
4 //! with parallel processing powered by Rayon.
5 //!
6 //! ## Architecture
7 //! - `api` - HTTP API handlers and server configuration
8 //! - `core` - Pure functions for text processing and similarity computation
9 //! - `models` - Immutable data structures
10 //! - `extraction` - File extraction modules (PDF, DOCX, TXT)
11 //! - `sentence` - Sentence splitting utilities
12
13 pub mod api;
14 pub mod core;
15 pub mod models;
16 pub mod extraction;
17 pub mod sentence;
18

localhost:33055/06e81d38-77a8-40c6-bafe-ec10aec40970/ 28/39
12/8/25, 8:45 AM src

src\[Link]

1 //! Document Similarity Analyzer - Main Entry Point

2 //!
3 //! A backend service for analyzing document similarity using TF-IDF
4 //! and Cosine Similarity with parallel processing.
5
6 use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
7
8 use document_similarity_­
analyzer::api::run_server;
9
10 #[tokio::main]
11 async fn main() -> anyhow::Result<()> {
12 // Initialize tracing/logging
13 tracing_subscriber::registry()
14 .with(
15 tracing_subscriber::EnvFilter::try_from_default_env­
()
16 .unwrap_or_else(|_| "document_similarity_­
‐
analyzer=debug,tower_http=debug".into()),
17 )
18 .with(tracing_subscriber::fmt::layer())
19 .init();
20
21 // Get port from environment or use default
22 let port: u16 = std::env::var("PORT")
23 .ok()
24 .and_then(|p| [Link]().ok())
25 .unwrap_or(3000);
26
27 // Run the server
28 run_server(port).await
29 }
30

localhost:33055/06e81d38-77a8-40c6-bafe-ec10aec40970/ 29/39
12/8/25, 8:45 AM src

src\models\[Link]

1 //! Document and processing data structures

2
3 use serde::{Deserialize, Serialize};
4 use std::collections::HashMap;
5
6 /// Represents a raw document with its ID
7 #[derive(Debug, Clone, Serialize, Deserialize)]
8 pub struct Document {
9 pub id: String,
10 pub content: String,
11 }
12
13 impl Document {
14 pub fn new(id: impl Into<String>, content: impl Into<String>) -> Self {
15 Self {
16 id: [Link](),
17 content: [Link](),
18 }
19 }
20 }
21
22 /// Represents a tokenized document
23 #[derive(Debug, Clone)]
24 pub struct TokenizedDoc {
25 pub id: String,
26 pub tokens: Vec<String>,
27 }
28
29 impl TokenizedDoc {
30 pub fn new(id: impl Into<String>, tokens: Vec<String>) -> Self {
31 Self {
32 id: [Link](),
33 tokens,
34 }
35 }
36 }
37
38 /// Term Frequency map for a single document
39 pub type TermFrequency = HashMap<String, f32>;
40
41 /// Inverse Document Frequency map across all documents
42 pub type InverseDocumentFrequ­
ency = HashMap<String, f32>;
43
44 /// TF-IDF Vector representation of a document
45 #[derive(Debug, Clone)]
46 pub struct TfIdfVector {
47 pub id: String,
48 pub vector: Vec<f32>,
49 }
50
51 impl TfIdfVector {

localhost:33055/06e81d38-77a8-40c6-bafe-ec10aec40970/ 30/39
12/8/25, 8:45 AM src

52 pub fn new(id: impl Into<String>, vector: Vec<f32>) -> Self {

53 Self {
54 id: [Link](),
55 vector,
56 }
57 }
58 }
59
60 /// Similarity matrix result
61 #[derive(Debug, Clone, Serialize, Deserialize)]
62 pub struct SimilarityMatrix {
63 /// NxN similarity matrix where matrix[i][j] is similarity between doc i and doc j
64 pub matrix: Vec<Vec<f32>>,
65 /// Document indices/labels
66 pub index: Vec<String>,
67 }
68
69 impl SimilarityMatrix {
70 pub fn new(matrix: Vec<Vec<f32>>, index: Vec<String>) -> Self {
71 Self { matrix, index }
72 }
73 }
74

localhost:33055/06e81d38-77a8-40c6-bafe-ec10aec40970/ 31/39
12/8/25, 8:45 AM src

src\models\[Link]

1 //! Immutable data models for document similarity analysis

2
3 mod document;
4 mod request;
5 mod response;
6 mod sentence_analysis;
7
8 pub use document::*;
9 pub use request::*;
10 pub use response::*;
11 pub use sentence_analysis::*;
12

localhost:33055/06e81d38-77a8-40c6-bafe-ec10aec40970/ 32/39
12/8/25, 8:45 AM src

src\models\[Link]

1 //! API request models

2
3 use serde::{Deserialize, Serialize};
4
5 /// Request payload for document analysis
6 #[derive(Debug, Clone, Serialize, Deserialize)]
7 pub struct AnalyzeRequest {
8 /// List of document texts to analyze
9 pub documents: Vec<String>,
10 }
11
12 impl AnalyzeRequest {
13 pub fn new(documents: Vec<String>) -> Self {
14 Self { documents }
15 }
16 }
17

localhost:33055/06e81d38-77a8-40c6-bafe-ec10aec40970/ 33/39
12/8/25, 8:45 AM src

src\models\[Link]

1 //! API response models

2
3 use serde::{Deserialize, Serialize};
4
5 /// Response payload for document analysis
6 #[derive(Debug, Clone, Serialize, Deserialize)]
7 pub struct AnalyzeResponse {
8 /// NxN similarity matrix
9 pub similarity_matrix: Vec<Vec<f32>>,
10 /// Document indices/labels
11 pub index: Vec<String>,
12 }
13
14 impl AnalyzeResponse {
15 pub fn new(similarity_matrix: Vec<Vec<f32>>, index: Vec<String>) -> Self {
16 Self {
17 similarity_matrix,
18 index,
19 }
20 }
21 }
22
23 impl From<crate::models::SimilarityMatrix> for AnalyzeResponse {
24 fn from(matrix: crate::models::SimilarityMatrix) -> Self {
25 Self {
26 similarity_matrix: [Link],
27 index: [Link],
28 }
29 }
30 }
31

localhost:33055/06e81d38-77a8-40c6-bafe-ec10aec40970/ 34/39
12/8/25, 8:45 AM src

src\models\sentence_analysis.rs

1 //! Models for sentence-level document analysis

2
3 use serde::{Deserialize, Serialize};
4
5 /// Metadata for analysis results
6 #[derive(Debug, Clone, Serialize, Deserialize)]
7 pub struct AnalysisMetadata {
8 /// Total number of documents analyzed
9 pub documents_count: usize,
10 /// Total number of sentences across all documents
11 pub total_sentences: usize,
12 /// Processing time in milliseconds
13 pub processing_time_ms: u64,
14 /// Similarity threshold used for filtering
15 pub threshold: f32,
16 }
17
18 impl AnalysisMetadata {
19 pub fn new(
20 documents_count: usize,
21 total_sentences: usize,
22 processing_time_ms: u64,
23 threshold: f32,
24 ) -> Self {
25 Self {
26 documents_count,
27 total_sentences,
28 processing_time_ms,
29 threshold,
30 }
31 }
32 }
33
34 /// A single sentence similarity match
35 #[derive(Debug, Clone, Serialize, Deserialize)]
36 pub struct SentenceMatch {
37 /// Source document filename
38 pub source_doc: String,
39 /// Index of sentence in source document (0-based)
40 pub source_sentence_inde­
x: usize,
41 /// The actual source sentence text
42 pub source_sentence: String,
43 /// Target document filename
44 pub target_doc: String,
45 /// Index of sentence in target document (0-based)
46 pub target_sentence_inde­
x: usize,
47 /// The actual target sentence text
48 pub target_sentence: String,
49 /// Similarity score (0.0 to 1.0)
50 pub similarity: f32,
51 }

localhost:33055/06e81d38-77a8-40c6-bafe-ec10aec40970/ 35/39
12/8/25, 8:45 AM src

52
53 impl SentenceMatch {
54 pub fn new(
55 source_doc: String,
56 source_sentence_inde­
x: usize,
57 source_sentence: String,
58 target_doc: String,
59 target_sentence_inde­
x: usize,
60 target_sentence: String,
61 similarity: f32,
62 ) -> Self {
63 Self {
64 source_doc,
65 source_sentence_inde­
x,
66 source_sentence,
67 target_doc,
68 target_sentence_inde­
x,
69 target_sentence,
70 similarity,
71 }
72 }
73 }
74
75 /// Global similarity between two documents
76 #[derive(Debug, Clone, Serialize, Deserialize)]
77 pub struct GlobalSimilarity {
78 /// First document filename
79 #[serde(rename = "docA")]
80 pub doc_a: String,
81 /// Second document filename
82 #[serde(rename = "docB")]
83 pub doc_b: String,
84 /// Overall similarity score (0.0 to 1.0)
85 pub score: f32,
86 }
87
88 impl GlobalSimilarity {
89 pub fn new(doc_a: String, doc_b: String, score: f32) -> Self {
90 Self { doc_a, doc_b, score }
91 }
92 }
93
94 /// Response payload for sentence-level document analysis
95 #[derive(Debug, Clone, Serialize, Deserialize)]
96 pub struct SentenceAnalysisResp­
onse {
97 /// Analysis metadata
98 pub metadata: AnalysisMetadata,
99 /// List of sentence matches above threshold
100 pub matches: Vec<SentenceMatch>,
101 /// Global similarity scores between document pairs
102 pub global_similarity: Vec<GlobalSimilarity>,
103 }
104
105 impl SentenceAnalysisResp­
onse {
localhost:33055/06e81d38-77a8-40c6-bafe-ec10aec40970/ 36/39
12/8/25, 8:45 AM src

106 pub fn new(

107 metadata: AnalysisMetadata,
108 matches: Vec<SentenceMatch>,
109 global_similarity: Vec<GlobalSimilarity>,
110 ) -> Self {
111 Self {
112 metadata,
113 matches,
114 global_similarity,
115 }
116 }
117 }
118

localhost:33055/06e81d38-77a8-40c6-bafe-ec10aec40970/ 37/39
12/8/25, 8:45 AM src

src\sentence\[Link]

1 //! Sentence splitting module

2
3 use lazy_static::lazy_static;
4 use regex::Regex;
5
6 lazy_static! {
7 /// Regex pattern for splitting sentences
8 /// Matches punctuation (. ! ?) followed by whitespace (including newlines)
9 /// or end of string
10 static ref SENTENCE_SPLITTER: Regex = Regex::new(r"[.!?](?:\s+|$)").unwrap();
11 }
12
13 /// Split text into sentences using regex
14 pub fn split_sentences(text: &str) -> Vec<String> {
15 if [Link]().is_empty() {
16 return vec![];
17 }
18
19 // Collect all matches with their positions
20 let matches: Vec<_> = SENTENCE_SPLITTER.find_iter(text).collect();
21
22 if matches.is_empty() {
23 // No sentence delimiters found, return entire text as one sentence
24 return vec![[Link]().to_string()].into_iter()
25 .filter(|s| !s.is_empty())
26 .collect();
27 }
28
29 // Build sentences from matches using functional approach
30 let sentence_ranges = [Link]().enumerate().map(|(idx, mat)| {
31 let start = if idx == 0 { 0 } else { matches[idx - 1].end() };
32 let end = [Link]() + 1; // +1 to include punctuation
33 (start, end)
34 });
35
36 // Collect sentences from ranges
37 let sentences_from_range­
s: Vec<String> = sentence_ranges
38 .map(|(start, end)| text[start..end].trim().to_string())
39 .filter(|s| !s.is_empty())
40 .collect();
41
42 // Add remaining text after last match if exists
43 let remaining_sentence = [Link]()
44 .and_then(|last_match| {
45 let remaining = text[last_match.end()..].trim();
46 if remaining.is_empty() {
47 None
48 } else {
49 Some(remaining.to_string())
50 }
51 });

localhost:33055/06e81d38-77a8-40c6-bafe-ec10aec40970/ 38/39
12/8/25, 8:45 AM src

52
53 // Combine sentences with optional remaining
54 sentences_from_range­
s.into_iter()
55 .chain(remaining_sentence)
56 .collect()
57 }
58

localhost:33055/06e81d38-77a8-40c6-bafe-ec10aec40970/ 39/39