Forge
search
csharp72c0d4fb
1using System.Text;
2using Microsoft.Data.Sqlite;
3
4namespace HybridCodebaseIndex.Core;
5
6internal static partial class SqliteFtsIndex
7{
8 internal static Task<(SearchResponse response, string? error)> SearchAsync(
9 string workspaceRoot,
10 string dbPath,
11 string query,
12 int topN,
13 string? pathPrefix,
14 IReadOnlyList<string>? excludePathPrefixes,
15 IReadOnlyList<string>? extensions,
16 CancellationToken cancellationToken)
17 => Task.Run(() => Search(workspaceRoot, dbPath, query, topN, pathPrefix, excludePathPrefixes, extensions), cancellationToken);
18
19 private static (SearchResponse response, string? error) Search(
20 string workspaceRoot,
21 string dbPath,
22 string userQuery,
23 int topN,
24 string? pathPrefix,
25 IReadOnlyList<string>? excludePathPrefixes,
26 IReadOnlyList<string>? extensions)
27 {
28 workspaceRoot = Path.GetFullPath(workspaceRoot.TrimEnd(Path.DirectorySeparatorChar));
29 if (!File.Exists(dbPath))
30 return (new SearchResponse(FormatVersion, userQuery, dbPath, []), "Index database not found; run codebase_index_reindex.");
31
32 var fts = BuildMatchQuery(userQuery);
33 if (fts is null)
34 return (new SearchResponse(FormatVersion, userQuery, dbPath, []), null);
35
36 using var conn = new SqliteConnection($"Data Source={dbPath};Mode=ReadOnly");
37 conn.Open();
38
39 static (SearchResponse response, string? error) RunQuery(SqliteConnection conn, string userQuery, string dbPath, string fts, int topN, string? pathPrefix, IReadOnlyList<string>? excludePathPrefixes, IReadOnlyList<string>? extensions, bool includeFileState)
40 {
41 using var cmd = conn.CreateCommand();
42 var sql = new StringBuilder();
43 sql.AppendLine(includeFileState
44 ? "SELECT chunks.rowid, chunks.path, chunks.extension, chunks.line_start, chunks.line_end, length(chunks.body), bm25(chunks), snippet(chunks, 4, '[', ']', ' … ', 24), fs.last_write_utc_ticks"
45 : "SELECT chunks.rowid, chunks.path, chunks.extension, chunks.line_start, chunks.line_end, length(chunks.body), bm25(chunks), snippet(chunks, 4, '[', ']', ' … ', 24), NULL");
46 sql.AppendLine("FROM chunks");
47 if (includeFileState)
48 sql.AppendLine("LEFT JOIN file_state fs ON fs.path = chunks.path");
49 sql.AppendLine("WHERE chunks MATCH $q");
50
51 if (!string.IsNullOrWhiteSpace(pathPrefix))
52 {
53 var pfx = NormalizePathPrefix(pathPrefix);
54 sql.AppendLine(" AND chunks.path LIKE $pfx ESCAPE '\\'");
55 cmd.Parameters.AddWithValue("$pfx", EscapeLike(pfx) + "%");
56 }
57
58 if (excludePathPrefixes is { Count: > 0 })
59 {
60 var i = 0;
61 foreach (var raw in excludePathPrefixes)
62 {
63 var p = NormalizePathPrefix(raw);
64 if (p.Length == 0)
65 continue;
66 var key = "$xp" + i++;
67 sql.AppendLine($" AND chunks.path NOT LIKE {key} ESCAPE '\\'");
68 cmd.Parameters.AddWithValue(key, EscapeLike(p) + "%");
69 }
70 }
71
72 if (extensions is { Count: > 0 })
73 {
74 var norm = NormalizeExtensionsForFilter(extensions);
75 if (norm.Count > 0)
76 {
77 var keys = new List<string>(norm.Count);
78 for (var i = 0; i < norm.Count; i++)
79 {
80 var k = "$e" + i;
81 keys.Add(k);
82 cmd.Parameters.AddWithValue(k, norm[i]);
83 }
84 sql.AppendLine($" AND extension IN ({string.Join(", ", keys)})");
85 }
86 }
87
88 sql.AppendLine("ORDER BY bm25(chunks) DESC");
89 sql.AppendLine("LIMIT $lim;");
90
91 cmd.CommandText = sql.ToString();
92 cmd.Parameters.AddWithValue("$q", fts);
93 cmd.Parameters.AddWithValue("$lim", topN);
94
95 var hits = new List<IndexHit>();
96 using var reader = cmd.ExecuteReader();
97 while (reader.Read())
98 {
99 var hitId = reader.GetInt64(0);
100 var path = reader.GetString(1);
101 var ext = reader.IsDBNull(2) ? "" : reader.GetString(2);
102 var lineStart = reader.IsDBNull(3) ? 0 : reader.GetInt32(3);
103 var lineEnd = reader.IsDBNull(4) ? 0 : reader.GetInt32(4);
104 var chunkChars = reader.IsDBNull(5) ? 0 : reader.GetInt32(5);
105 var bm = reader.GetDouble(6);
106 var snip = reader.IsDBNull(7) ? null : reader.GetString(7);
107 var lastWriteIso = reader.IsDBNull(8)
108 ? null
109 : new DateTime(reader.GetInt64(8), DateTimeKind.Utc).ToString("O");
110 hits.Add(new IndexHit(hitId, path, ext, HitKinds.TextFts, bm, FtsScore: bm, VecScore: null, snip, lineStart, lineEnd, chunkChars, lastWriteIso));
111 }
112
113 return (new SearchResponse(FormatVersion, userQuery, dbPath, hits), null);
114 }
115
116 try
117 {
118 return RunQuery(conn, userQuery, dbPath, fts, topN, pathPrefix, excludePathPrefixes, extensions, includeFileState: true);
119 }
120 catch (SqliteException ex) when (ex.SqliteErrorCode == 1 && ex.Message.Contains("file_state", StringComparison.OrdinalIgnoreCase))
121 {
122 // Back-compat: older DBs may not have file_state. Fall back without freshness.
123 return RunQuery(conn, userQuery, dbPath, fts, topN, pathPrefix, excludePathPrefixes, extensions, includeFileState: false);
124 }
125 }
126
127 private static string NormalizePathPrefix(string raw)
128 => raw.Trim().Replace("\\", "/", StringComparison.Ordinal).TrimStart('/');
129
130 private static string EscapeLike(string value)
131 => value.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("%", "\\%", StringComparison.Ordinal).Replace("_", "\\_", StringComparison.Ordinal);
132
133 private static List<string> NormalizeExtensionsForFilter(IReadOnlyList<string> raw)
134 {
135 var list = new List<string>(raw.Count);
136 foreach (var s0 in raw)
137 {
138 var s = s0.Trim();
139 if (s.Length == 0)
140 continue;
141 if (!s.StartsWith(".", StringComparison.Ordinal))
142 s = "." + s;
143 s = s.ToLowerInvariant();
144 list.Add(s);
145 }
146 return list;
147 }
148
149 internal static Task<ExplainHitResponse> ExplainHitAsync(
150 string workspaceRoot,
151 string dbPath,
152 long hitId,
153 CancellationToken cancellationToken)
154 => Task.Run(() => ExplainHit(workspaceRoot, dbPath, hitId), cancellationToken);
155
156 private static ExplainHitResponse ExplainHit(string workspaceRoot, string dbPath, long hitId)
157 {
158 workspaceRoot = Path.GetFullPath(workspaceRoot.TrimEnd(Path.DirectorySeparatorChar));
159 if (!File.Exists(dbPath))
160 return new ExplainHitResponse(FormatVersion, dbPath, null, "Index database not found; run codebase_index_reindex.");
161
162 using var conn = new SqliteConnection($"Data Source={dbPath};Mode=ReadOnly");
163 conn.Open();
164
165 using var cmd = conn.CreateCommand();
166 cmd.CommandText = """
167 SELECT c.rowid, c.path, c.extension, c.line_start, c.line_end, length(c.body), substr(c.body, 1, 1200), fs.last_write_utc_ticks
168 FROM chunks c
169 LEFT JOIN file_state fs ON fs.path = c.path
170 WHERE c.rowid = $id
171 LIMIT 1;
172 """;
173 cmd.Parameters.AddWithValue("$id", hitId);
174
175 using var r = cmd.ExecuteReader();
176 if (!r.Read())
177 return new ExplainHitResponse(FormatVersion, dbPath, null, $"Hit not found: {hitId}");
178
179 var id = r.GetInt64(0);
180 var path = r.GetString(1);
181 var ext = r.IsDBNull(2) ? "" : r.GetString(2);
182 var lineStart = r.IsDBNull(3) ? 0 : r.GetInt32(3);
183 var lineEnd = r.IsDBNull(4) ? 0 : r.GetInt32(4);
184 var chunkChars = r.IsDBNull(5) ? 0 : r.GetInt32(5);
185 var body = r.IsDBNull(6) ? null : r.GetString(6);
186 var lastWriteIso = r.IsDBNull(7)
187 ? null
188 : new DateTime(r.GetInt64(7), DateTimeKind.Utc).ToString("O");
189
190 var hit = new IndexHit(id, path, ext, HitKinds.TextFts, 0, FtsScore: null, VecScore: null, body, lineStart, lineEnd, chunkChars, lastWriteIso);
191 return new ExplainHitResponse(FormatVersion, dbPath, hit, null);
192 }
193
194 /// <summary>Безопасное FTS5 MATCH: токены через AND, суффикс * (префиксное совпадение). Пустой запрос → null.</summary>
195 internal static string? BuildMatchQuery(string userQuery)
196 {
197 var tokens = userQuery.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
198 .Select(static t => t.Trim().Trim('"'))
199 .Where(static t => t.Length > 0)
200 .Take(24)
201 .ToArray();
202 if (tokens.Length == 0)
203 return null;
204
205 var parts = new List<string>(tokens.Length);
206 foreach (var t in tokens)
207 {
208 var safe = t.Replace("\"", "", StringComparison.Ordinal).Replace("'", "", StringComparison.Ordinal);
209 if (safe.Length == 0)
210 continue;
211 parts.Add('"' + safe + "\"*");
212 }
213
214 return parts.Count == 0 ? null : string.Join(" AND ", parts);
215 }
216}
217
218
View only · write via MCP/CIDE