Forge
search
csharp72c0d4fb
1using System.Globalization;
2using System.Text;
3using HybridCodebaseIndex.Core.Indexing;
4using Microsoft.Data.Sqlite;
5
6namespace HybridCodebaseIndex.Core;
7
8internal static partial class SqliteFtsIndex
9{
10 private static ReindexSummary ReindexIncremental(
11 string workspaceRoot,
12 string dbPath,
13 IReadOnlyList<ICodebaseIndexReindexObserver>? reindexObservers,
14 CancellationToken cancellationToken)
15 {
16 var sw = System.Diagnostics.Stopwatch.StartNew();
17 workspaceRoot = Path.GetFullPath(workspaceRoot.TrimEnd(Path.DirectorySeparatorChar));
18
19 var filesIndexed = 0;
20 var skippedLarge = 0;
21 var skippedBinary = 0;
22 var skippedExcluded = 0;
23 var skippedSample = new List<SkippedPath>(capacity: 64);
24 var skippedReasonCounts = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);
25 var skippedTopPathPrefixes = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);
26
27 using var conn = new SqliteConnection($"Data Source={dbPath};Mode=ReadWriteCreate");
28 conn.Open();
29
30 EnsureMetaTable(conn);
31 EnsureChunksTable(conn);
32 EnsureFileStateTable(conn);
33
34 try
35 {
36 UpsertMeta(conn, "reindex_state", "running");
37 UpsertMeta(conn, "reindex_started_at", DateTime.UtcNow.ToString("O", CultureInfo.InvariantCulture));
38
39 var settings = IndexSettings.TryLoadFromIndexDirectory(Path.GetDirectoryName(dbPath)!);
40 var extensions = settings.GetEffectiveExtensions();
41 var maxBytes = settings.GetEffectiveMaxIndexedFileBytes();
42 var chunkLines = settings.GetEffectiveChunkLines();
43 var overlapLines = settings.GetEffectiveChunkOverlapLines();
44 var probeBytes = settings.GetEffectiveBinaryProbeBytes();
45
46 var roots = new List<string>(capacity: 1 + settings.ExtraIncludeRoots.Count) { workspaceRoot };
47 foreach (var extra in settings.ExtraIncludeRoots)
48 {
49 var p = Path.GetFullPath(Path.Combine(workspaceRoot, extra));
50 if (Directory.Exists(p))
51 roots.Add(p);
52 }
53
54 var extSet = new HashSet<string>(extensions, StringComparer.OrdinalIgnoreCase);
55
56 var excludeRootFullPaths = new List<string>(settings.ExcludeRoots.Count);
57 foreach (var rel in settings.ExcludeRoots)
58 {
59 var p = rel?.Trim();
60 if (string.IsNullOrWhiteSpace(p))
61 continue;
62 if (Path.IsPathRooted(p) || p.Contains("..", StringComparison.Ordinal))
63 continue;
64 var abs = Path.GetFullPath(Path.Combine(workspaceRoot, p));
65 if (Directory.Exists(abs))
66 excludeRootFullPaths.Add(abs.TrimEnd(Path.DirectorySeparatorChar) + Path.DirectorySeparatorChar);
67 }
68
69 var candidates = new List<string>(capacity: 8192);
70 foreach (var root in roots)
71 candidates.AddRange(WorkspaceScanner.EnumerateIndexableFiles(root, extSet, settings.ExcludePathSegments, excludeRootFullPaths));
72
73 var gitIgnore = GitIgnoreRules.TryLoad(workspaceRoot, settings.IgnoreFiles);
74
75 var seen = new HashSet<string>(StringComparer.Ordinal);
76
77 using var tx = conn.BeginTransaction();
78
79 foreach (var absolute in candidates)
80 {
81 cancellationToken.ThrowIfCancellationRequested();
82
83 if (WorkspaceScanner.ShouldExcludePath(absolute, settings.ExcludePathSegments))
84 {
85 skippedExcluded++;
86 AddSkip(skippedSample, skippedReasonCounts, skippedTopPathPrefixes, WorkspaceScanner.RelativePath(workspaceRoot, absolute), "denylist");
87 continue;
88 }
89
90 var rel = WorkspaceScanner.RelativePath(workspaceRoot, absolute).Replace("\\", "/", StringComparison.Ordinal);
91 seen.Add(rel);
92
93 if (GitIgnoreRules.IsIgnored(gitIgnore, rel))
94 {
95 skippedExcluded++;
96 AddSkip(skippedSample, skippedReasonCounts, skippedTopPathPrefixes, rel, "gitignore");
97 continue;
98 }
99
100 FileInfo info;
101 try
102 {
103 info = new FileInfo(absolute);
104 if (info.Length > maxBytes)
105 {
106 skippedLarge++;
107 AddSkip(skippedSample, skippedReasonCounts, skippedTopPathPrefixes, rel, "too_large");
108 continue;
109 }
110 }
111 catch
112 {
113 skippedExcluded++;
114 AddSkip(skippedSample, skippedReasonCounts, skippedTopPathPrefixes, rel, "io_error");
115 continue;
116 }
117
118 var lastWriteUtcTicks = info.LastWriteTimeUtc.Ticks;
119 if (!IsFileChanged(conn, tx, rel, info.Length, lastWriteUtcTicks))
120 continue;
121
122 using var fs = info.OpenRead();
123 var probeSize = (int)Math.Min(probeBytes, info.Length);
124 var probe = new byte[probeSize];
125 var read = fs.ReadAtLeast(probe.AsSpan(0, probeSize), probeSize, throwOnEndOfStream: false);
126 if (WorkspaceScanner.LooksBinary(probe.AsSpan(0, read)))
127 {
128 skippedBinary++;
129 AddSkip(skippedSample, skippedReasonCounts, skippedTopPathPrefixes, rel, "binary");
130 continue;
131 }
132
133 fs.Position = 0;
134 using var reader = new StreamReader(fs, Encoding.UTF8, detectEncodingFromByteOrderMarks: true);
135 var text = reader.ReadToEnd();
136
137 // Replace chunks for this path.
138 DeleteChunksForPath(conn, tx, rel);
139
140 var ext = Path.GetExtension(absolute);
141 NotifyFileIndexed(reindexObservers, workspaceRoot, rel, absolute, ext, text, lastWriteUtcTicks);
142 var header = BuildArtifactAugmentationHeader(workspaceRoot, absolute, rel, ext, text);
143 var chunks = WorkspaceScanner.ChunkByLines(text, chunkLines, overlapLines);
144 var anyChunk = false;
145 foreach (var (lineStart, lineEnd, body) in chunks)
146 {
147 var augmentedBody = lineStart == 1 && header.Length > 0 ? header + body : body;
148 InsertChunk(conn, tx, rel, ext, lineStart, lineEnd, augmentedBody);
149 anyChunk = true;
150 }
151
152 if (anyChunk)
153 filesIndexed++;
154
155 UpsertFileState(conn, tx, rel, info.Length, lastWriteUtcTicks);
156 }
157
158 // Remove deleted files.
159 foreach (var stale in EnumerateStalePaths(conn, tx, seen))
160 {
161 DeleteChunksForPath(conn, tx, stale);
162 DeleteFileState(conn, tx, stale);
163 }
164
165 tx.Commit();
166
167 UpsertMeta(conn, "format_version", FormatVersion.ToString(CultureInfo.InvariantCulture));
168 UpsertMeta(conn, "indexed_at", DateTime.UtcNow.ToString("O", CultureInfo.InvariantCulture));
169 UpsertMeta(conn, "workspace_root", workspaceRoot);
170 UpsertMeta(conn, "reindex_error", "");
171 UpsertMeta(conn, "reindex_error_at", "");
172 UpsertMeta(conn, "reindex_state", "idle");
173 UpsertMeta(conn, "reindex_started_at", "");
174 }
175 catch (Exception ex)
176 {
177 try
178 {
179 UpsertMeta(conn, "reindex_state", "error");
180 UpsertMeta(conn, "reindex_error", ex.GetType().Name + ": " + ex.Message);
181 UpsertMeta(conn, "reindex_error_at", DateTime.UtcNow.ToString("O", CultureInfo.InvariantCulture));
182 }
183 catch
184 {
185 // ignore
186 }
187
188 throw;
189 }
190
191 sw.Stop();
192 return new ReindexSummary(
193 FormatVersion,
194 dbPath,
195 filesIndexed,
196 skippedLarge,
197 skippedBinary,
198 skippedExcluded,
199 skippedReasonCounts,
200 TopPrefixes(skippedTopPathPrefixes),
201 skippedSample,
202 sw.Elapsed);
203 }
204}
205
206
View only · write via MCP/CIDE