| 1 | using System.Text.Json; |
| 2 | using Tomlyn; |
| 3 | |
| 4 | namespace HybridCodebaseIndex.Core; |
| 5 | |
| 6 | public sealed record IndexSettings( |
| 7 | bool IncludeCsInFts, |
| 8 | IReadOnlyList<string> ExtraIncludeRoots, |
| 9 | IReadOnlyList<string> ExcludeRoots, |
| 10 | IReadOnlyList<string>? IncludeExtensions, |
| 11 | IReadOnlyList<string>? ExcludeExtensions, |
| 12 | IReadOnlyList<string> ExcludePathSegments, |
| 13 | IReadOnlyList<string> IgnoreFiles, |
| 14 | long MaxIndexedFileBytes, |
| 15 | int ChunkLines, |
| 16 | int ChunkOverlapLines, |
| 17 | int BinaryProbeBytes, |
| 18 | bool SemanticEnabled, |
| 19 | string? EmbeddingProvider, |
| 20 | string? EmbeddingModel, |
| 21 | int EmbeddingDim, |
| 22 | string? EmbeddingModelPath, |
| 23 | string? EmbeddingVocabPath, |
| 24 | bool EmbeddingDoLowerCase, |
| 25 | string? VecExtensionsMode, |
| 26 | IReadOnlyList<string>? VecExtensions, |
| 27 | IReadOnlyList<string>? VecAddExtensions, |
| 28 | IReadOnlyList<string>? VecRemoveExtensions, |
| 29 | string? SqliteVecExtensionPath, |
| 30 | int EmbeddingSequenceLength, |
| 31 | bool EmbeddingPreferGpu) |
| 32 | { |
| 33 | private static class HciTomlSerializer |
| 34 | { |
| 35 | private static readonly TomlSerializerOptions Options = new() |
| 36 | { |
| 37 | PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, |
| 38 | }; |
| 39 | |
| 40 | public static T? Deserialize<T>(string text) => TomlSerializer.Deserialize<T>(text, Options); |
| 41 | } |
| 42 | |
| 43 | private sealed class SettingsTomlRoot |
| 44 | { |
| 45 | public ScopeToml? Scope { get; set; } |
| 46 | public FtsToml? Fts { get; set; } |
| 47 | public SemanticToml? Semantic { get; set; } |
| 48 | } |
| 49 | |
| 50 | private sealed class ScopeToml |
| 51 | { |
| 52 | public List<string>? ExtraIncludeRoots { get; set; } |
| 53 | public List<string>? ExcludeRoots { get; set; } |
| 54 | public List<string>? ExcludePathSegments { get; set; } |
| 55 | public List<string>? IgnoreFiles { get; set; } |
| 56 | } |
| 57 | |
| 58 | private sealed class FtsToml |
| 59 | { |
| 60 | public bool? IncludeCsInFts { get; set; } |
| 61 | public List<string>? IncludeExtensions { get; set; } |
| 62 | public List<string>? ExcludeExtensions { get; set; } |
| 63 | public long? MaxIndexedFileBytes { get; set; } |
| 64 | public int? ChunkLines { get; set; } |
| 65 | public int? ChunkOverlapLines { get; set; } |
| 66 | public int? BinaryProbeBytes { get; set; } |
| 67 | } |
| 68 | |
| 69 | private sealed class SemanticToml |
| 70 | { |
| 71 | public bool? Enabled { get; set; } |
| 72 | public EmbeddingsToml? Embeddings { get; set; } |
| 73 | |
| 74 | public string? SqliteVecExtensionPath { get; set; } |
| 75 | public string? VecExtensionsMode { get; set; } |
| 76 | public List<string>? VecExtensions { get; set; } |
| 77 | public List<string>? VecAddExtensions { get; set; } |
| 78 | public List<string>? VecRemoveExtensions { get; set; } |
| 79 | } |
| 80 | |
| 81 | private sealed class EmbeddingsToml |
| 82 | { |
| 83 | public string? Provider { get; set; } |
| 84 | public string? Model { get; set; } |
| 85 | public int? Dim { get; set; } |
| 86 | public string? ModelPath { get; set; } |
| 87 | public string? VocabPath { get; set; } |
| 88 | public bool? DoLowerCase { get; set; } |
| 89 | public int? SequenceLength { get; set; } |
| 90 | public bool? PreferGpu { get; set; } |
| 91 | } |
| 92 | |
| 93 | /// <summary> |
| 94 | /// Если merge настроек не дал ни одного расширения, используем тот же состав, что во встроенном |
| 95 | /// <c>DefaultSettings/settings.default.toml</c> (без дублирования комментариев в коде — только список). |
| 96 | /// </summary> |
| 97 | private static readonly string[] FallbackIndexedExtensions = |
| 98 | [ |
| 99 | ".md", ".mdx", ".csproj", ".slnx", ".props", ".targets", ".toml", ".editorconfig", ".json", ".yml", |
| 100 | ".yaml", ".razor", ".css", ".scss", ".html", ".axaml", ".cs", |
| 101 | ]; |
| 102 | |
| 103 | public static IndexSettings Default { get; } = new( |
| 104 | IncludeCsInFts: true, |
| 105 | ExtraIncludeRoots: [], |
| 106 | ExcludeRoots: [], |
| 107 | IncludeExtensions: null, |
| 108 | ExcludeExtensions: null, |
| 109 | ExcludePathSegments: [], |
| 110 | IgnoreFiles: [], |
| 111 | MaxIndexedFileBytes: 0, |
| 112 | ChunkLines: 0, |
| 113 | ChunkOverlapLines: 0, |
| 114 | BinaryProbeBytes: 0, |
| 115 | SemanticEnabled: false, |
| 116 | EmbeddingProvider: null, |
| 117 | EmbeddingModel: null, |
| 118 | EmbeddingDim: 0, |
| 119 | EmbeddingModelPath: null, |
| 120 | EmbeddingVocabPath: null, |
| 121 | EmbeddingDoLowerCase: true, |
| 122 | VecExtensionsMode: "inherit_fts", |
| 123 | VecExtensions: null, |
| 124 | VecAddExtensions: null, |
| 125 | VecRemoveExtensions: null, |
| 126 | SqliteVecExtensionPath: null, |
| 127 | EmbeddingSequenceLength: 0, |
| 128 | EmbeddingPreferGpu: true); |
| 129 | |
| 130 | public static IndexSettings TryLoadFromIndexDirectory(string? indexDirectory) |
| 131 | { |
| 132 | _ = TryLoadFromIndexDirectoryWithDiagnostics(indexDirectory, out var settings, out _, out _); |
| 133 | return settings; |
| 134 | } |
| 135 | |
| 136 | public static bool TryLoadFromIndexDirectoryWithDiagnostics( |
| 137 | string? indexDirectory, |
| 138 | out IndexSettings settings, |
| 139 | out string settingsSource, |
| 140 | out string? settingsParseError) |
| 141 | { |
| 142 | settings = Default; |
| 143 | settingsSource = "default"; |
| 144 | settingsParseError = null; |
| 145 | |
| 146 | if (string.IsNullOrWhiteSpace(indexDirectory)) |
| 147 | return false; |
| 148 | |
| 149 | var dir = Path.GetFullPath(indexDirectory.TrimEnd(Path.DirectorySeparatorChar)); |
| 150 | var path = Path.Combine(dir, "settings.toml"); |
| 151 | |
| 152 | var embeddedModel = TryReadEmbeddedModel(out var embeddedErr); |
| 153 | string? diskErr = null; |
| 154 | var diskModel = File.Exists(path) ? TryReadDiskModel(path, out diskErr) : null; |
| 155 | |
| 156 | settingsParseError = diskErr ?? embeddedErr; |
| 157 | |
| 158 | if (embeddedModel is not null && diskModel is not null) |
| 159 | settingsSource = "embedded+disk"; |
| 160 | else if (diskModel is not null) |
| 161 | settingsSource = "disk"; |
| 162 | else if (embeddedModel is not null) |
| 163 | settingsSource = "embedded"; |
| 164 | |
| 165 | var merged = MergeModels(baseModel: embeddedModel, overlay: diskModel) ?? new SettingsTomlRoot(); |
| 166 | |
| 167 | settings = FromMergedToml(merged); |
| 168 | return true; |
| 169 | } |
| 170 | |
| 171 | private static IndexSettings FromMergedToml(SettingsTomlRoot merged) |
| 172 | { |
| 173 | var includeCs = merged.Fts?.IncludeCsInFts ?? Default.IncludeCsInFts; |
| 174 | |
| 175 | var extraRoots = merged.Scope?.ExtraIncludeRoots ?? []; |
| 176 | var excludeRoots = merged.Scope?.ExcludeRoots ?? []; |
| 177 | var excludeSegments = merged.Scope?.ExcludePathSegments ?? []; |
| 178 | var ignoreFiles = merged.Scope?.IgnoreFiles ?? []; |
| 179 | |
| 180 | var includeExt = NormalizeExtensions(merged.Fts?.IncludeExtensions); |
| 181 | var excludeExt = NormalizeExtensions(merged.Fts?.ExcludeExtensions); |
| 182 | |
| 183 | var maxBytes = merged.Fts?.MaxIndexedFileBytes ?? Default.MaxIndexedFileBytes; |
| 184 | var chunkLines = merged.Fts?.ChunkLines ?? Default.ChunkLines; |
| 185 | var overlapLines = merged.Fts?.ChunkOverlapLines ?? Default.ChunkOverlapLines; |
| 186 | var probeBytes = merged.Fts?.BinaryProbeBytes ?? Default.BinaryProbeBytes; |
| 187 | |
| 188 | var semanticEnabled = merged.Semantic?.Enabled ?? Default.SemanticEnabled; |
| 189 | var embeddings = merged.Semantic?.Embeddings; |
| 190 | var embeddingProvider = embeddings?.Provider ?? Default.EmbeddingProvider; |
| 191 | var embeddingModel = embeddings?.Model ?? Default.EmbeddingModel; |
| 192 | var embeddingDim = embeddings?.Dim ?? Default.EmbeddingDim; |
| 193 | var embeddingModelPath = embeddings?.ModelPath ?? Default.EmbeddingModelPath; |
| 194 | var embeddingVocabPath = embeddings?.VocabPath ?? Default.EmbeddingVocabPath; |
| 195 | var embeddingDoLowerCase = embeddings?.DoLowerCase ?? Default.EmbeddingDoLowerCase; |
| 196 | var embeddingSeqLen = embeddings?.SequenceLength ?? Default.EmbeddingSequenceLength; |
| 197 | var embeddingPreferGpu = embeddings?.PreferGpu ?? Default.EmbeddingPreferGpu; |
| 198 | |
| 199 | var vecExtensionsMode = merged.Semantic?.VecExtensionsMode ?? Default.VecExtensionsMode; |
| 200 | var vecExtensions = NormalizeExtensions(merged.Semantic?.VecExtensions); |
| 201 | var vecAddExtensions = NormalizeExtensions(merged.Semantic?.VecAddExtensions); |
| 202 | var vecRemoveExtensions = NormalizeExtensions(merged.Semantic?.VecRemoveExtensions); |
| 203 | var sqliteVecExtensionPath = merged.Semantic?.SqliteVecExtensionPath ?? Default.SqliteVecExtensionPath; |
| 204 | |
| 205 | return new IndexSettings( |
| 206 | includeCs, |
| 207 | extraRoots, |
| 208 | excludeRoots, |
| 209 | includeExt, |
| 210 | excludeExt, |
| 211 | excludeSegments, |
| 212 | ignoreFiles, |
| 213 | maxBytes, |
| 214 | chunkLines, |
| 215 | overlapLines, |
| 216 | probeBytes, |
| 217 | semanticEnabled, |
| 218 | embeddingProvider, |
| 219 | embeddingModel, |
| 220 | embeddingDim, |
| 221 | embeddingModelPath, |
| 222 | embeddingVocabPath, |
| 223 | embeddingDoLowerCase, |
| 224 | vecExtensionsMode, |
| 225 | vecExtensions, |
| 226 | vecAddExtensions, |
| 227 | vecRemoveExtensions, |
| 228 | sqliteVecExtensionPath, |
| 229 | embeddingSeqLen, |
| 230 | embeddingPreferGpu); |
| 231 | } |
| 232 | |
| 233 | public long GetEffectiveMaxIndexedFileBytes() |
| 234 | => MaxIndexedFileBytes > 0 ? MaxIndexedFileBytes : 512 * 1024; |
| 235 | |
| 236 | public int GetEffectiveChunkLines() |
| 237 | => ChunkLines > 0 ? ChunkLines : 110; |
| 238 | |
| 239 | public int GetEffectiveChunkOverlapLines() |
| 240 | => ChunkOverlapLines > 0 ? ChunkOverlapLines : 15; |
| 241 | |
| 242 | public int GetEffectiveBinaryProbeBytes() |
| 243 | => BinaryProbeBytes > 0 ? BinaryProbeBytes : 8192; |
| 244 | |
| 245 | public IReadOnlyList<string> GetEffectiveExtensions() |
| 246 | { |
| 247 | IReadOnlyList<string> baseList; |
| 248 | if (IncludeExtensions is { Count: > 0 }) |
| 249 | { |
| 250 | baseList = IncludeExtensions; |
| 251 | } |
| 252 | else |
| 253 | { |
| 254 | // Пустой/null после merge: см. MergeFts — overlay `include_extensions = []` не затирает базу. |
| 255 | // Если база тоже недоступна (редкий сбой embedded), не оставляем индекс без расширений. |
| 256 | baseList = FallbackIndexedExtensions; |
| 257 | } |
| 258 | |
| 259 | IEnumerable<string> filtered = baseList; |
| 260 | if (ExcludeExtensions is { Count: > 0 }) |
| 261 | { |
| 262 | var deny = new HashSet<string>(ExcludeExtensions, StringComparer.OrdinalIgnoreCase); |
| 263 | filtered = filtered.Where(e => !deny.Contains(e)); |
| 264 | } |
| 265 | |
| 266 | // include_cs_in_fts is a first-class toggle; treat it as stronger than include_extensions defaults. |
| 267 | if (!IncludeCsInFts) |
| 268 | filtered = filtered.Where(static e => !string.Equals(e, ".cs", StringComparison.OrdinalIgnoreCase)); |
| 269 | else if (baseList.Count > 0 && !filtered.Contains(".cs", StringComparer.OrdinalIgnoreCase)) |
| 270 | filtered = filtered.Concat([".cs"]); |
| 271 | |
| 272 | return filtered.ToArray(); |
| 273 | } |
| 274 | |
| 275 | /// <summary>Те же правила, что <see cref="GetEffectiveExtensions"/>, для vec и проверок на границе поиска.</summary> |
| 276 | public HashSet<string> GetEffectiveExtensionsSet() |
| 277 | => new(GetEffectiveExtensions(), StringComparer.OrdinalIgnoreCase); |
| 278 | |
| 279 | /// <summary> |
| 280 | /// Эффективные расширения для vec. |
| 281 | /// По умолчанию наследует effective extensions от FTS и применяет add/remove. |
| 282 | /// При <c>vec_extensions_mode="custom"</c> может использовать базовый список <c>semantic.vec_extensions</c>. |
| 283 | /// </summary> |
| 284 | public HashSet<string> GetEffectiveVecExtensionsSet() |
| 285 | { |
| 286 | HashSet<string> set; |
| 287 | |
| 288 | var mode = (VecExtensionsMode ?? "inherit_fts").Trim().ToLowerInvariant(); |
| 289 | if (string.Equals(mode, "custom", StringComparison.OrdinalIgnoreCase) && VecExtensions is { Count: > 0 }) |
| 290 | set = new HashSet<string>(VecExtensions, StringComparer.OrdinalIgnoreCase); |
| 291 | else |
| 292 | set = GetEffectiveExtensionsSet(); |
| 293 | |
| 294 | if (VecRemoveExtensions is { Count: > 0 }) |
| 295 | foreach (var ext in VecRemoveExtensions) |
| 296 | set.Remove(ext); |
| 297 | |
| 298 | if (VecAddExtensions is { Count: > 0 }) |
| 299 | foreach (var ext in VecAddExtensions) |
| 300 | set.Add(ext); |
| 301 | |
| 302 | return set; |
| 303 | } |
| 304 | |
| 305 | private static SettingsTomlRoot? TryReadEmbeddedModel(out string? error) |
| 306 | { |
| 307 | error = null; |
| 308 | try |
| 309 | { |
| 310 | if (!BundledContent.TryReadEmbeddedText("DefaultSettings/settings.default.toml", out var embedded)) |
| 311 | return null; |
| 312 | return HciTomlSerializer.Deserialize<SettingsTomlRoot>(embedded); |
| 313 | } |
| 314 | catch (Exception ex) |
| 315 | { |
| 316 | error = ex.GetType().Name + ": " + ex.Message; |
| 317 | return null; |
| 318 | } |
| 319 | } |
| 320 | |
| 321 | private static SettingsTomlRoot? TryReadDiskModel(string path, out string? error) |
| 322 | { |
| 323 | error = null; |
| 324 | try |
| 325 | { |
| 326 | var text = File.ReadAllText(path); |
| 327 | return HciTomlSerializer.Deserialize<SettingsTomlRoot>(text); |
| 328 | } |
| 329 | catch (Exception ex) |
| 330 | { |
| 331 | error = ex.GetType().Name + ": " + ex.Message; |
| 332 | return null; |
| 333 | } |
| 334 | } |
| 335 | |
| 336 | private static List<string>? NormalizeExtensions(List<string>? raw) |
| 337 | { |
| 338 | if (raw is null || raw.Count == 0) |
| 339 | return raw; |
| 340 | |
| 341 | var list = new List<string>(raw.Count); |
| 342 | foreach (var s0 in raw) |
| 343 | { |
| 344 | var s = s0.Trim(); |
| 345 | if (s.Length == 0) |
| 346 | continue; |
| 347 | if (!s.StartsWith(".", StringComparison.Ordinal)) |
| 348 | s = "." + s; |
| 349 | list.Add(s.ToLowerInvariant()); |
| 350 | } |
| 351 | return list; |
| 352 | } |
| 353 | |
| 354 | private static SettingsTomlRoot? MergeModels(SettingsTomlRoot? baseModel, SettingsTomlRoot? overlay) |
| 355 | { |
| 356 | if (baseModel is null && overlay is null) |
| 357 | return null; |
| 358 | if (baseModel is null) |
| 359 | return overlay; |
| 360 | if (overlay is null) |
| 361 | return baseModel; |
| 362 | |
| 363 | return new SettingsTomlRoot |
| 364 | { |
| 365 | Scope = MergeScope(baseModel.Scope, overlay.Scope), |
| 366 | Fts = MergeFts(baseModel.Fts, overlay.Fts), |
| 367 | Semantic = MergeSemantic(baseModel.Semantic, overlay.Semantic), |
| 368 | }; |
| 369 | } |
| 370 | |
| 371 | private static ScopeToml? MergeScope(ScopeToml? a, ScopeToml? b) |
| 372 | { |
| 373 | if (a is null && b is null) |
| 374 | return null; |
| 375 | a ??= new ScopeToml(); |
| 376 | b ??= new ScopeToml(); |
| 377 | return new ScopeToml |
| 378 | { |
| 379 | ExtraIncludeRoots = b.ExtraIncludeRoots ?? a.ExtraIncludeRoots, |
| 380 | ExcludeRoots = b.ExcludeRoots ?? a.ExcludeRoots, |
| 381 | ExcludePathSegments = b.ExcludePathSegments ?? a.ExcludePathSegments, |
| 382 | IgnoreFiles = b.IgnoreFiles ?? a.IgnoreFiles, |
| 383 | }; |
| 384 | } |
| 385 | |
| 386 | private static FtsToml? MergeFts(FtsToml? a, FtsToml? b) |
| 387 | { |
| 388 | if (a is null && b is null) |
| 389 | return null; |
| 390 | a ??= new FtsToml(); |
| 391 | b ??= new FtsToml(); |
| 392 | return new FtsToml |
| 393 | { |
| 394 | IncludeCsInFts = b.IncludeCsInFts ?? a.IncludeCsInFts, |
| 395 | // Пустой массив в overlay не должен затирать встроенный список (иначе индексируется 0 файлов). |
| 396 | IncludeExtensions = b.IncludeExtensions is { Count: > 0 } ? b.IncludeExtensions : a.IncludeExtensions, |
| 397 | ExcludeExtensions = b.ExcludeExtensions ?? a.ExcludeExtensions, |
| 398 | MaxIndexedFileBytes = b.MaxIndexedFileBytes ?? a.MaxIndexedFileBytes, |
| 399 | ChunkLines = b.ChunkLines ?? a.ChunkLines, |
| 400 | ChunkOverlapLines = b.ChunkOverlapLines ?? a.ChunkOverlapLines, |
| 401 | BinaryProbeBytes = b.BinaryProbeBytes ?? a.BinaryProbeBytes, |
| 402 | }; |
| 403 | } |
| 404 | |
| 405 | private static SemanticToml? MergeSemantic(SemanticToml? a, SemanticToml? b) |
| 406 | { |
| 407 | if (a is null && b is null) |
| 408 | return null; |
| 409 | a ??= new SemanticToml(); |
| 410 | b ??= new SemanticToml(); |
| 411 | return new SemanticToml |
| 412 | { |
| 413 | Enabled = b.Enabled ?? a.Enabled, |
| 414 | Embeddings = MergeEmbeddings(a.Embeddings, b.Embeddings), |
| 415 | SqliteVecExtensionPath = b.SqliteVecExtensionPath ?? a.SqliteVecExtensionPath, |
| 416 | VecExtensionsMode = b.VecExtensionsMode ?? a.VecExtensionsMode, |
| 417 | VecExtensions = b.VecExtensions ?? a.VecExtensions, |
| 418 | VecAddExtensions = b.VecAddExtensions ?? a.VecAddExtensions, |
| 419 | VecRemoveExtensions = b.VecRemoveExtensions ?? a.VecRemoveExtensions, |
| 420 | }; |
| 421 | } |
| 422 | |
| 423 | private static EmbeddingsToml? MergeEmbeddings(EmbeddingsToml? a, EmbeddingsToml? b) |
| 424 | { |
| 425 | if (a is null && b is null) |
| 426 | return null; |
| 427 | a ??= new EmbeddingsToml(); |
| 428 | b ??= new EmbeddingsToml(); |
| 429 | return new EmbeddingsToml |
| 430 | { |
| 431 | Provider = b.Provider ?? a.Provider, |
| 432 | Model = b.Model ?? a.Model, |
| 433 | Dim = b.Dim ?? a.Dim, |
| 434 | ModelPath = b.ModelPath ?? a.ModelPath, |
| 435 | VocabPath = b.VocabPath ?? a.VocabPath, |
| 436 | DoLowerCase = b.DoLowerCase ?? a.DoLowerCase, |
| 437 | SequenceLength = b.SequenceLength ?? a.SequenceLength, |
| 438 | PreferGpu = b.PreferGpu ?? a.PreferGpu, |
| 439 | }; |
| 440 | } |
| 441 | } |
| 442 | |
| 443 | |