Forge
search
csharpdeeb25a2
1#nullable enable
2
3using System.Buffers;
4using System.Net;
5using System.Net.Http.Headers;
6using System.Net.Sockets;
7using System.Text;
8using System.Text.Json;
9using System.Text.RegularExpressions;
10
11namespace CascadeIDE.Services;
12
13/// <summary>
14/// Одиночный HTTP GET по HTTPS с лимитами размера; грубое извлечение текста из HTML (аналог «Fetch» для агента).
15/// </summary>
16internal static partial class WebPublicDocumentFetchClient
17{
18 internal const int DefaultMaxChars = 200_000;
19 private const int DefaultMaxDownloadBytes = 2 * 1024 * 1024;
20
21 private static readonly HttpClient Http = CreateHttpClient();
22
23 private static readonly JsonSerializerOptions ResponseJsonOpts = new()
24 {
25 PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
26 WriteIndented = false,
27 };
28
29 private static HttpClient CreateHttpClient()
30 {
31 var http = new HttpClient { Timeout = TimeSpan.FromSeconds(45) };
32 http.DefaultRequestVersion = HttpVersion.Version20;
33 http.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", "CascadeIDE/1.0 (fetch-web-public-url)");
34 http.DefaultRequestHeaders.TryAddWithoutValidation("Accept", "text/html,text/plain,application/json,*/*;q=0.8");
35 return http;
36 }
37
38 /// <returns>JSON: url, resolved_url, status_code, content_type, text, truncated_download, truncated_text, extraction, error?, note.</returns>
39 internal static async Task<string> FetchAsync(string urlRaw, int? maxChars, CancellationToken cancellationToken)
40 {
41 var maxOut = maxChars is > 0 and <= 1_000_000 ? maxChars.Value : DefaultMaxChars;
42
43 var trimmed = urlRaw?.Trim() ?? "";
44 if (trimmed.Length == 0)
45 return ErrorJson(trimmed, null, "empty url", "Передай непустой absolute https URL.");
46
47 if (!Uri.TryCreate(trimmed, UriKind.Absolute, out var uri))
48 return ErrorJson(trimmed, null, "invalid url", "Нужен абсолютный URL.");
49
50 if (!string.Equals(uri.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase))
51 return ErrorJson(trimmed, uri.ToString(), "https only", "Разрешена только схема https.");
52
53 var host = uri.IdnHost;
54 if (IsBlockedEndpointHost(host))
55 return ErrorJson(trimmed, uri.ToString(), "host not allowed",
56 "Локальные и частные узлы недоступны (базовая защита от SSRF).");
57
58 try
59 {
60 using var request = new HttpRequestMessage(HttpMethod.Get, uri);
61 using var response = await Http.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
62
63 var resolved = response.RequestMessage?.RequestUri?.ToString() ?? uri.ToString();
64 var status = (int)response.StatusCode;
65
66 await using var networkStream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
67 var (body, truncatedDl) = await ReadCappedAsync(networkStream, DefaultMaxDownloadBytes, cancellationToken).ConfigureAwait(false);
68
69 if (!response.IsSuccessStatusCode)
70 {
71 var preview = TrimForPreview(body.Length == 0 ? "" : ProbeDecode(body));
72 return JsonSerializer.Serialize(new
73 {
74 url = trimmed,
75 resolved_url = resolved,
76 status_code = status,
77 content_type = ContentTypeBrief(response.Content.Headers.ContentType),
78 text = "",
79 truncated_download = truncatedDl,
80 truncated_text = false,
81 extraction = "none",
82 error = $"HTTP {status}",
83 body_preview = preview,
84 note = "Для ошибок протокола смотри status_code и body_preview.",
85 }, ResponseJsonOpts);
86 }
87
88 var mediaType = response.Content.Headers.ContentType?.MediaType ?? "";
89 var charset = response.Content.Headers.ContentType?.CharSet;
90 var decoded = DecodeBody(body, charset);
91 string text;
92 var extraction = "plain";
93
94 if (LooksLikeHtml(decoded, mediaType))
95 {
96 text = HtmlToReadableText(decoded);
97 extraction = "html_to_text";
98 }
99 else
100 {
101 text = decoded;
102 if (mediaType.Contains("json", StringComparison.OrdinalIgnoreCase))
103 extraction = "json_as_text";
104 }
105
106 NormalizeWhitespace(ref text);
107
108 var truncatedText = text.Length > maxOut;
109 if (truncatedText)
110 text = text[..maxOut] + "…";
111
112 return JsonSerializer.Serialize(new
113 {
114 url = trimmed,
115 resolved_url = resolved,
116 status_code = status,
117 content_type = ContentTypeBrief(response.Content.Headers.ContentType),
118 text,
119 truncated_download = truncatedDl,
120 truncated_text = truncatedText,
121 extraction,
122 note = "Упрощённое извлечение из HTML; сложная вёрстка и скрипты могут дать шум. Редиректы следуются — resolved_url может отличаться.",
123 }, ResponseJsonOpts);
124 }
125 catch (OperationCanceledException)
126 {
127 throw;
128 }
129 catch (Exception ex)
130 {
131 return ErrorJson(trimmed, trimmed, ex.Message, null);
132 }
133 }
134
135 private static string ErrorJson(string url, string? resolvedUrl, string error, string? note) =>
136 JsonSerializer.Serialize(new
137 {
138 url,
139 resolved_url = resolvedUrl,
140 error,
141 text = "",
142 truncated_download = false,
143 truncated_text = false,
144 extraction = "none",
145 note,
146 }, ResponseJsonOpts);
147
148 private static bool IsBlockedEndpointHost(string host)
149 {
150 if (host.Length == 0)
151 return true;
152
153 if (string.Equals(host, "localhost", StringComparison.OrdinalIgnoreCase))
154 return true;
155
156 if (IPAddress.TryParse(host, out var ip))
157 return IsPrivateOrLocalIp(ip);
158
159 return false;
160 }
161
162 private static bool IsPrivateOrLocalIp(IPAddress ip)
163 {
164 if (IPAddress.IsLoopback(ip))
165 return true;
166
167 if (ip.AddressFamily == AddressFamily.InterNetwork)
168 {
169 var b = ip.GetAddressBytes();
170 if (b[0] == 10)
171 return true;
172 if (b[0] == 127)
173 return true;
174 if (b[0] == 0)
175 return true;
176 if (b[0] == 169 && b[1] == 254)
177 return true;
178 if (b[0] == 172 && b[1] >= 16 && b[1] <= 31)
179 return true;
180 if (b[0] == 192 && b[1] == 168)
181 return true;
182 }
183
184 if (ip.AddressFamily == AddressFamily.InterNetworkV6)
185 {
186 if (ip.IsIPv6LinkLocal)
187 return true;
188 // Unique local (fc00::/7)
189 var s = ip.ToString();
190 if (s.StartsWith("fc", StringComparison.OrdinalIgnoreCase) || s.StartsWith("fd", StringComparison.OrdinalIgnoreCase))
191 return true;
192 }
193
194 return false;
195 }
196
197 private static async Task<(byte[] Body, bool Truncated)> ReadCappedAsync(Stream stream, int maxBytes, CancellationToken ct)
198 {
199 var rent = ArrayPool<byte>.Shared.Rent(8192);
200 try
201 {
202 await using var ms = new MemoryStream(capacity: Math.Min(maxBytes, 65536));
203 var total = 0;
204 while (true)
205 {
206 var read = await stream.ReadAsync(rent.AsMemory(0, rent.Length), ct).ConfigureAwait(false);
207 if (read == 0)
208 break;
209 var next = total + read;
210 if (next > maxBytes)
211 {
212 var allow = maxBytes - total;
213 if (allow > 0)
214 ms.Write(rent, 0, allow);
215 return (ms.ToArray(), true);
216 }
217
218 ms.Write(rent, 0, read);
219 total = next;
220 }
221
222 return (ms.ToArray(), false);
223 }
224 finally
225 {
226 ArrayPool<byte>.Shared.Return(rent);
227 }
228 }
229
230 private static string ProbeDecode(byte[] body) => DecodeBody(body, null);
231
232 private static string DecodeBody(byte[] body, string? headerCharset)
233 {
234 if (body.Length == 0)
235 return "";
236
237 Encoding? enc = null;
238 if (!string.IsNullOrWhiteSpace(headerCharset))
239 {
240 try { enc = Encoding.GetEncoding(headerCharset); }
241 catch (ArgumentException) { /* use UTF-8 */ }
242 }
243
244 if (enc is null)
245 {
246 if (body.Length >= 3 && body[0] == 0xEF && body[1] == 0xBB && body[2] == 0xBF)
247 return Encoding.UTF8.GetString(body.AsSpan(3));
248 if (body.Length >= 2 && body[0] == 0xFF && body[1] == 0xFE)
249 return Encoding.Unicode.GetString(body.AsSpan(2));
250 return Encoding.UTF8.GetString(body);
251 }
252
253 return enc.GetString(body);
254 }
255
256 private static string ContentTypeBrief(MediaTypeHeaderValue? mt)
257 {
258 if (mt is null)
259 return "";
260 var s = mt.MediaType ?? "";
261 if (!string.IsNullOrEmpty(mt.CharSet))
262 s += "; charset=" + mt.CharSet;
263 return s;
264 }
265
266 private static bool LooksLikeHtml(string s, string mediaType)
267 {
268 if (mediaType.Contains("html", StringComparison.OrdinalIgnoreCase))
269 return true;
270 var t = s.AsSpan().TrimStart();
271 return t.StartsWith("<!DOCTYPE", StringComparison.OrdinalIgnoreCase)
272 || t.StartsWith("<html", StringComparison.OrdinalIgnoreCase)
273 || t.StartsWith("<HTML", StringComparison.Ordinal);
274 }
275
276 private static string HtmlToReadableText(string html)
277 {
278 var s = html;
279
280 s = ScriptStyleRx().Replace(s, " ");
281 s = BlockCloseRx().Replace(s, "\n");
282 s = TagRx().Replace(s, " ");
283 s = WebUtility.HtmlDecode(s);
284 return s;
285 }
286
287 private static void NormalizeWhitespace(ref string s)
288 {
289 s = LineBreakRx().Replace(s, "\n");
290 s = MultiSpaceRx().Replace(s, " ");
291 s = MultiNlRx().Replace(s, "\n\n");
292 s = s.Trim();
293 }
294
295 private static string TrimForPreview(string s, int max = 400) =>
296 s.Length <= max ? s : s[..max] + "…";
297
298 [GeneratedRegex(@"<script\b[^>]*>.*?</script>|<style\b[^>]*>.*?</style>", RegexOptions.IgnoreCase | RegexOptions.Singleline)]
299 private static partial Regex ScriptStyleRx();
300
301 [GeneratedRegex(@"</(p|div|br|tr|h[1-6]|li|section|article|header|footer|main)\b[^>]*>", RegexOptions.IgnoreCase)]
302 private static partial Regex BlockCloseRx();
303
304 [GeneratedRegex(@"<[^>]+>", RegexOptions.Singleline)]
305 private static partial Regex TagRx();
306
307 [GeneratedRegex(@"\r\n|\r")]
308 private static partial Regex LineBreakRx();
309
310 [GeneratedRegex(@"[ \t]{2,}")]
311 private static partial Regex MultiSpaceRx();
312
313 [GeneratedRegex(@"\n{3,}")]
314 private static partial Regex MultiNlRx();
315}
316
View only · write via MCP/CIDE