Forge
search
csharp72c0d4fb
1using System.Globalization;
2using System.Text;
3using HybridCodebaseIndex.Core.Indexing;
4using Microsoft.Data.Sqlite;
5
6namespace HybridCodebaseIndex.Core;
7
8internal static partial class SqliteFtsIndex
9{
10 private static ReindexSummary FullRebuild(
11 string workspaceRoot,
12 string dbPath,
13 IReadOnlyList<ICodebaseIndexReindexObserver>? reindexObservers,
14 CancellationToken cancellationToken)
15 {
16 var sw = System.Diagnostics.Stopwatch.StartNew();
17 workspaceRoot = Path.GetFullPath(workspaceRoot.TrimEnd(Path.DirectorySeparatorChar));
18
19 var filesIndexed = 0;
20 var skippedLarge = 0;
21 var skippedBinary = 0;
22 var skippedExcluded = 0;
23 var skippedSample = new List<SkippedPath>(capacity: 64);
24 var skippedReasonCounts = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);
25 var skippedTopPathPrefixes = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);
26
27 using var conn = new SqliteConnection($"Data Source={dbPath};Mode=ReadWriteCreate");
28 conn.Open();
29
30 // Root cause fix for Windows file-locking: do NOT replace/rename the DB file.
31 // Rebuild in-place by swapping tables inside the same SQLite file (WAL allows concurrent readers).
32 EnsureMetaTable(conn);
33 EnsureFileStateTable(conn);
34
35 try
36 {
37 UpsertMeta(conn, "reindex_state", "running");
38 UpsertMeta(conn, "reindex_started_at", DateTime.UtcNow.ToString("O", CultureInfo.InvariantCulture));
39
40 var settings = IndexSettings.TryLoadFromIndexDirectory(Path.GetDirectoryName(dbPath)!);
41 var extensions = settings.GetEffectiveExtensions();
42 var maxBytes = settings.GetEffectiveMaxIndexedFileBytes();
43 var chunkLines = settings.GetEffectiveChunkLines();
44 var overlapLines = settings.GetEffectiveChunkOverlapLines();
45 var probeBytes = settings.GetEffectiveBinaryProbeBytes();
46
47 // Prepare a fresh FTS table for the new build.
48 Exec(conn, "DROP TABLE IF EXISTS chunks_new;");
49 Exec(conn, """
50 CREATE VIRTUAL TABLE chunks_new USING fts5(
51 path,
52 extension UNINDEXED,
53 line_start UNINDEXED,
54 line_end UNINDEXED,
55 body,
56 tokenize='unicode61 remove_diacritics 1'
57 );
58 """);
59
60 using var tx = conn.BeginTransaction();
61
62 // Rebuild file_state from scratch for freshness metadata.
63 Exec(conn, "DELETE FROM file_state;", tx);
64 using var insert = conn.CreateCommand();
65 insert.Transaction = tx;
66 insert.CommandText = """
67 INSERT INTO chunks_new(path, extension, line_start, line_end, body)
68 VALUES ($path, $ext, $ls, $le, $body);
69 """;
70
71 var roots = new List<string>(capacity: 1 + settings.ExtraIncludeRoots.Count) { workspaceRoot };
72 foreach (var extra in settings.ExtraIncludeRoots)
73 {
74 var p = Path.GetFullPath(Path.Combine(workspaceRoot, extra));
75 if (Directory.Exists(p))
76 roots.Add(p);
77 }
78
79 var extSet = new HashSet<string>(extensions, StringComparer.OrdinalIgnoreCase);
80
81 var excludeRootFullPaths = new List<string>(settings.ExcludeRoots.Count);
82 foreach (var rel in settings.ExcludeRoots)
83 {
84 var p = rel?.Trim();
85 if (string.IsNullOrWhiteSpace(p))
86 continue;
87 if (Path.IsPathRooted(p) || p.Contains("..", StringComparison.Ordinal))
88 continue;
89 var abs = Path.GetFullPath(Path.Combine(workspaceRoot, p));
90 if (Directory.Exists(abs))
91 excludeRootFullPaths.Add(abs.TrimEnd(Path.DirectorySeparatorChar) + Path.DirectorySeparatorChar);
92 }
93
94 var candidates = new List<string>(capacity: 8192);
95 foreach (var root in roots)
96 candidates.AddRange(WorkspaceScanner.EnumerateIndexableFiles(root, extSet, settings.ExcludePathSegments, excludeRootFullPaths));
97
98 var gitIgnore = GitIgnoreRules.TryLoad(workspaceRoot, settings.IgnoreFiles);
99
100 foreach (var absolute in candidates)
101 {
102 cancellationToken.ThrowIfCancellationRequested();
103
104 if (WorkspaceScanner.ShouldExcludePath(absolute, settings.ExcludePathSegments))
105 {
106 skippedExcluded++;
107 AddSkip(skippedSample, skippedReasonCounts, skippedTopPathPrefixes, WorkspaceScanner.RelativePath(workspaceRoot, absolute), "denylist");
108 continue;
109 }
110 }
111
112 foreach (var absolute in candidates)
113 {
114 cancellationToken.ThrowIfCancellationRequested();
115
116 if (WorkspaceScanner.ShouldExcludePath(absolute, settings.ExcludePathSegments))
117 continue;
118
119 var rel = WorkspaceScanner.RelativePath(workspaceRoot, absolute).Replace("\\", "/", StringComparison.Ordinal);
120 if (GitIgnoreRules.IsIgnored(gitIgnore, rel))
121 {
122 skippedExcluded++;
123 AddSkip(skippedSample, skippedReasonCounts, skippedTopPathPrefixes, rel, "gitignore");
124 continue;
125 }
126
127 FileInfo info;
128 try
129 {
130 info = new FileInfo(absolute);
131 if (info.Length > maxBytes)
132 {
133 skippedLarge++;
134 AddSkip(skippedSample, skippedReasonCounts, skippedTopPathPrefixes, rel, "too_large");
135 continue;
136 }
137 }
138 catch
139 {
140 skippedExcluded++;
141 AddSkip(skippedSample, skippedReasonCounts, skippedTopPathPrefixes, rel, "io_error");
142 continue;
143 }
144
145 using var fs = info.OpenRead();
146 var probeSize = (int)Math.Min(probeBytes, info.Length);
147 var probe = new byte[probeSize];
148 var read = fs.ReadAtLeast(probe.AsSpan(0, probeSize), probeSize, throwOnEndOfStream: false);
149 if (WorkspaceScanner.LooksBinary(probe.AsSpan(0, read)))
150 {
151 skippedBinary++;
152 AddSkip(skippedSample, skippedReasonCounts, skippedTopPathPrefixes, rel, "binary");
153 continue;
154 }
155
156 fs.Position = 0;
157 using var reader = new StreamReader(fs, Encoding.UTF8, detectEncodingFromByteOrderMarks: true);
158 var text = reader.ReadToEnd();
159
160 var ext = Path.GetExtension(absolute);
161 var lastWriteUtcTicks = info.LastWriteTimeUtc.Ticks;
162 NotifyFileIndexed(reindexObservers, workspaceRoot, rel, absolute, ext, text, lastWriteUtcTicks);
163 var header = BuildArtifactAugmentationHeader(workspaceRoot, absolute, rel, ext, text);
164
165 var chunks = WorkspaceScanner.ChunkByLines(text, chunkLines, overlapLines);
166 var anyChunk = false;
167 foreach (var (lineStart, lineEnd, body) in chunks)
168 {
169 var augmentedBody = lineStart == 1 && header.Length > 0 ? header + body : body;
170 insert.Parameters.Clear();
171 insert.Parameters.AddWithValue("$path", rel);
172 insert.Parameters.AddWithValue("$ext", ext);
173 insert.Parameters.AddWithValue("$ls", lineStart);
174 insert.Parameters.AddWithValue("$le", lineEnd);
175 insert.Parameters.AddWithValue("$body", augmentedBody);
176 insert.ExecuteNonQuery();
177 anyChunk = true;
178 }
179
180 if (anyChunk)
181 {
182 filesIndexed++;
183 UpsertFileState(conn, tx, rel, info.Length, lastWriteUtcTicks);
184 }
185 }
186
187 tx.Commit();
188
189 // Swap tables atomically.
190 using (var swapTx = conn.BeginTransaction())
191 {
192 // Keep it simple: run DDL under a single transaction object, avoid manual BEGIN/COMMIT SQL.
193 Exec(conn, "DROP TABLE IF EXISTS chunks_old;", swapTx);
194 try
195 {
196 Exec(conn, "ALTER TABLE chunks RENAME TO chunks_old;", swapTx);
197 }
198 catch (SqliteException)
199 {
200 // First build: chunks doesn't exist yet.
201 }
202
203 Exec(conn, "ALTER TABLE chunks_new RENAME TO chunks;", swapTx);
204 Exec(conn, "DROP TABLE IF EXISTS chunks_old;", swapTx);
205 swapTx.Commit();
206 }
207
208 UpsertMeta(conn, "format_version", FormatVersion.ToString(CultureInfo.InvariantCulture));
209 UpsertMeta(conn, "indexed_at", DateTime.UtcNow.ToString("O", CultureInfo.InvariantCulture));
210 UpsertMeta(conn, "workspace_root", workspaceRoot);
211
212 // Clear last error on success.
213 UpsertMeta(conn, "reindex_error", "");
214 UpsertMeta(conn, "reindex_error_at", "");
215
216 UpsertMeta(conn, "reindex_state", "idle");
217 UpsertMeta(conn, "reindex_started_at", "");
218 }
219 catch (Exception ex)
220 {
221 // Best-effort: store the last failure so status can surface it.
222 try
223 {
224 UpsertMeta(conn, "reindex_state", "error");
225 UpsertMeta(conn, "reindex_error", ex.GetType().Name + ": " + ex.Message);
226 UpsertMeta(conn, "reindex_error_at", DateTime.UtcNow.ToString("O", CultureInfo.InvariantCulture));
227 // keep started_at as-is
228 }
229 catch
230 {
231 // ignore
232 }
233
234 throw;
235 }
236
237 sw.Stop();
238
239 return new ReindexSummary(
240 FormatVersion,
241 dbPath,
242 filesIndexed,
243 skippedLarge,
244 skippedBinary,
245 skippedExcluded,
246 skippedReasonCounts,
247 TopPrefixes(skippedTopPathPrefixes),
248 skippedSample,
249 sw.Elapsed);
250 }
251}
252
253
View only · write via MCP/CIDE