Supporto per aste chiuse e miglioramenti UI

- Aggiornamento alla versione Microsoft.EntityFrameworkCore.Sqlite 8.0.0.
- Aggiornamento alla versione Microsoft.Windows.SDK.BuildTools 10.0.26100.6584.
- Migliorata l'interfaccia per l'inserimento di più URL/ID di aste.
- Aggiunti pulsanti per "Aste Chiuse" e "Esporta" in MainWindow.
- Creata finestra "Aste Chiuse" per visualizzare e gestire aste chiuse.
- Implementato scraper per estrarre dati da aste chiuse.
- Aggiunto supporto per esportazione dati in CSV, JSON e XML.
- Introdotto contesto Entity Framework per statistiche delle aste.
- Aggiunto servizio per calcolo e gestione delle statistiche.
- Gestite preferenze di esportazione con salvataggio in file JSON.
This commit is contained in:
Alberto Balbo
2025-11-03 14:24:19 +01:00
parent 59d7e0c41f
commit 967005b96a
13 changed files with 1128 additions and 42 deletions
+370
View File
@@ -0,0 +1,370 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using AutoBidder.Models;
namespace AutoBidder.Services
{
/// <summary>
/// Semplice scraper che scarica la pagina delle "closed auctions" di Bidoo,
/// estrae i link alle singole aste, visita ciascuna pagina e prova ad estrarre
/// informazioni utili (nome prodotto, prezzo finale, vincitore, puntate usate).
/// Risultato salvato in CSV per analisi statistiche esterne.
///
/// Nota: il parsing è basato su euristiche (regex) per resistere a vari formati HTML.
/// </summary>
public class ClosedAuctionsScraper
{
private readonly HttpClient _http;
private readonly StatsService? _statsService;
private readonly Action<string>? _log;
public ClosedAuctionsScraper(HttpMessageHandler? handler = null, StatsService? statsService = null, Action<string>? log = null)
{
var h = handler ?? new HttpClientHandler
{
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate | DecompressionMethods.Brotli
};
_http = new HttpClient(h)
{
Timeout = TimeSpan.FromSeconds(10)
};
// Default headers user-like
_http.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36");
_http.DefaultRequestHeaders.TryAddWithoutValidation("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
_http.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Language", "it-IT,it;q=0.9,en-US;q=0.8,en;q=0.7");
_statsService = statsService;
_log = log;
}
/// <summary>
/// Scarica la pagina di aste chiuse, estrae i link ed esegue scraping per ogni asta.
/// Salva il risultato in CSV.
/// </summary>
public async Task ScrapeAndSaveCsvAsync(string closedAuctionsUrl, string outputCsvPath)
{
var results = await ScrapeAsync(closedAuctionsUrl);
SaveCsv(results, outputCsvPath);
}
/// <summary>
/// Scarica la pagina di aste chiuse, estrae i link ed esegue scraping per ogni asta.
/// Ritorna la lista dei record (non salva su disco).
/// </summary>
public async Task<List<ClosedAuctionRecord>> ScrapeAsync(string closedAuctionsUrl)
{
var list = new List<ClosedAuctionRecord>();
await foreach (var rec in ScrapeYieldAsync(closedAuctionsUrl))
{
list.Add(rec);
}
return list;
}
/// <summary>
/// Scarica la pagina di aste chiuse e produce i record uno per uno (yield) per permettere aggiornamenti UI incrementali.
/// </summary>
public async IAsyncEnumerable<ClosedAuctionRecord> ScrapeYieldAsync(string closedAuctionsUrl)
{
if (string.IsNullOrWhiteSpace(closedAuctionsUrl)) throw new ArgumentNullException(nameof(closedAuctionsUrl));
var baseUri = new Uri("https://it.bidoo.com/");
_log?.Invoke($"[scraper] Downloading closed auctions page: {closedAuctionsUrl}");
var html = await GetStringAsync(closedAuctionsUrl);
if (html == null)
{
_log?.Invoke("[scraper] ERROR: unable to download closed auctions page");
throw new InvalidOperationException("Impossibile scaricare la pagina delle aste chiuse.");
}
var auctionUrls = ExtractAuctionLinks(html, baseUri).Distinct().ToList();
_log?.Invoke($"[scraper] Found {auctionUrls.Count} auction links on closed auctions page.");
foreach (var auctionUrl in auctionUrls)
{
ClosedAuctionRecord record;
try
{
_log?.Invoke($"[scraper] Fetching auction page: {auctionUrl}");
var contextInfo = ExtractSummaryInfoForUrl(html, auctionUrl);
var auctionHtml = await GetStringAsync(auctionUrl);
if (auctionHtml == null)
{
_log?.Invoke($"[scraper] WARNING: failed to download auction page: {auctionUrl}");
throw new InvalidOperationException("Download auction page failed");
}
var productName = contextInfo?.ProductName ?? ExtractProductNameFromAuctionHtml(auctionHtml);
var finalPrice = contextInfo?.FinalPrice ?? ExtractFinalPriceFromAuctionHtml(auctionHtml);
var winner = contextInfo?.Winner ?? ExtractWinnerFromAuctionHtml(auctionHtml);
var bidsUsed = ExtractBidsUsedFromAuctionHtml(auctionHtml);
_log?.Invoke($"[scraper] Parsed: Name='{productName}', FinalPrice={(finalPrice.HasValue? finalPrice.Value.ToString("F2", CultureInfo.InvariantCulture):"null")}, Winner='{winner}', BidsUsed={(bidsUsed.HasValue?bidsUsed.Value.ToString():"null")}" );
// Ensure HTML entities decoded already by helper methods
record = new ClosedAuctionRecord
{
AuctionUrl = auctionUrl,
ProductName = productName,
FinalPrice = finalPrice,
Winner = winner,
BidsUsed = bidsUsed,
ScrapedAt = DateTime.UtcNow,
Notes = string.Empty
};
// Record stats if service provided (fire-and-forget)
if (_statsService != null)
{
#pragma warning disable CS4014
_statsService.RecordClosedAuctionAsync(record);
#pragma warning restore CS4014
}
}
catch (Exception ex)
{
_log?.Invoke($"[scraper] ERROR parsing auction {auctionUrl}: {ex.Message}");
record = new ClosedAuctionRecord
{
AuctionUrl = auctionUrl,
ProductName = "(parse error)",
FinalPrice = null,
Winner = null,
BidsUsed = null,
ScrapedAt = DateTime.UtcNow,
Notes = ex.Message
};
}
yield return record;
}
}
private async Task<string?> GetStringAsync(string url)
{
try
{
var uri = new Uri(url, UriKind.RelativeOrAbsolute);
if (!uri.IsAbsoluteUri)
{
uri = new Uri(new Uri("https://it.bidoo.com"), url);
}
var req = new HttpRequestMessage(HttpMethod.Get, uri);
req.Headers.TryAddWithoutValidation("Referer", "https://it.bidoo.com/");
var resp = await _http.SendAsync(req);
resp.EnsureSuccessStatusCode();
var txt = await resp.Content.ReadAsStringAsync();
_log?.Invoke($"[scraper] HTTP {resp.StatusCode} {uri}");
return txt;
}
catch (Exception ex)
{
_log?.Invoke($"[scraper] HTTP ERROR fetching {url}: {ex.Message}");
return null;
}
}
private IEnumerable<string> ExtractAuctionLinks(string closedHtml, Uri baseUri)
{
var urls = new List<string>();
// Cerca attributi data-href
var mh = Regex.Matches(closedHtml, "data-href\\s*=\\s*\\\"(?<u>[^\\\"]+)\\\"", RegexOptions.IgnoreCase);
foreach (Match m in mh)
{
var u = m.Groups["u"].Value.Trim();
if (!string.IsNullOrEmpty(u)) urls.Add(ToAbsolute(u, baseUri));
}
// fallback: cerca link a auction.php?a=
var mh2 = Regex.Matches(closedHtml, "href\\s*=\\s*\\\"(?<u>[^\\\"]*auction.php\\?a=[^\\\"]+)\\\"", RegexOptions.IgnoreCase);
foreach (Match m in mh2)
{
var u = m.Groups["u"].Value.Trim();
urls.Add(ToAbsolute(u, baseUri));
}
return urls.Where(u => !string.IsNullOrWhiteSpace(u));
}
private string ToAbsolute(string url, Uri baseUri)
{
try
{
if (url.StartsWith("http://", StringComparison.OrdinalIgnoreCase) || url.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
return url;
if (url.StartsWith("//"))
return "https:" + url;
if (url.StartsWith("/"))
return new Uri(baseUri, url).ToString();
return new Uri(baseUri, "/" + url).ToString();
}
catch
{
return url;
}
}
private (string? ProductName, double? FinalPrice, string? Winner)? ExtractSummaryInfoForUrl(string closedHtml, string auctionUrl)
{
try
{
var idx = closedHtml.IndexOf(auctionUrl, StringComparison.OrdinalIgnoreCase);
if (idx < 0) return null;
var start = Math.Max(0, idx - 800);
var len = Math.Min(2500, closedHtml.Length - start);
var seg = closedHtml.Substring(start, len);
var namePattern1 = "<b[^>]*class=\\\"media-heading\\\"[^>]*>\\s*<a[^>]*>(?<name>[^<]+)</a>";
var namePattern2 = "<span[^>]*class=\\\"media-heading[^\\\"]*\\\"[^>]*>\\s*<a[^>]*>(?<name>[^<]+)</a>";
var nameMatch = Regex.Match(seg, namePattern1, RegexOptions.IgnoreCase);
if (!nameMatch.Success)
{
nameMatch = Regex.Match(seg, namePattern2, RegexOptions.IgnoreCase);
}
var product = nameMatch.Success ? WebUtility.HtmlDecode(nameMatch.Groups["name"].Value).Trim() : null;
var priceMatch = Regex.Match(seg, "<span[^>]*class=\\\"price\\\"[^>]*>(?<p>[0-9.,]+)\\s*€", RegexOptions.IgnoreCase);
double? price = null;
if (priceMatch.Success) price = ParseEuro(priceMatch.Groups["p"].Value);
var winnerPattern1 = "<span[^>]*class=\\\"username\\\"[^>]*>.*?<span[^>]*class=\\\"offer\\\"[^>]*>(?<w>[^<]+)</span>";
var winnerPattern2 = "<span[^>]*class=\\\"mobile_offerer offer\\\"[^>]*>(?<w>[^<]+)</span>";
var winnerMatch = Regex.Match(seg, winnerPattern1, RegexOptions.IgnoreCase | RegexOptions.Singleline);
if (!winnerMatch.Success)
{
winnerMatch = Regex.Match(seg, winnerPattern2, RegexOptions.IgnoreCase);
}
var winner = winnerMatch.Success ? WebUtility.HtmlDecode(winnerMatch.Groups["w"].Value).Trim() : null;
return (product, price, winner);
}
catch
{
return null;
}
}
private string? ExtractProductNameFromAuctionHtml(string? auctionHtml)
{
if (string.IsNullOrEmpty(auctionHtml)) return null;
var content = auctionHtml ?? string.Empty;
var m = Regex.Match(content, "<h1[^>]*>(?<n>.*?)</h1>", RegexOptions.IgnoreCase | RegexOptions.Singleline);
if (m.Success) return WebUtility.HtmlDecode(StripTags(m.Groups["n"].Value)).Trim();
m = Regex.Match(content, "<title[^>]*>(?<t>.*?)</title>", RegexOptions.IgnoreCase | RegexOptions.Singleline);
if (m.Success) return WebUtility.HtmlDecode(StripTags(m.Groups["t"].Value)).Trim();
m = Regex.Match(content, "<b[^>]*class=\\\"media-heading\\\"[^>]*>\\s*<a[^>]*>(?<name>[^<]+)</a>", RegexOptions.IgnoreCase);
if (m.Success) return WebUtility.HtmlDecode(m.Groups["name"].Value).Trim();
return null;
}
private double? ExtractFinalPriceFromAuctionHtml(string? auctionHtml)
{
if (string.IsNullOrEmpty(auctionHtml)) return null;
var content = auctionHtml ?? string.Empty;
var m = Regex.Match(content, "<span[^>]*class=\\\"price\\\"[^>]*>(?<p>[0-9.,]+)\\s*€", RegexOptions.IgnoreCase);
if (m.Success) return ParseEuro(m.Groups["p"].Value);
m = Regex.Match(content, "prez[zo]?[\\\"\\']?[^0-9]{0,30}(?<p>[0-9.,]+)\\s*€", RegexOptions.IgnoreCase);
if (m.Success) return ParseEuro(m.Groups["p"].Value);
m = Regex.Match(content, "([0-9]{1,3}(?:[.,][0-9]{2}))\\s*€", RegexOptions.IgnoreCase);
if (m.Success) return ParseEuro(m.Groups[1].Value);
return null;
}
private string? ExtractWinnerFromAuctionHtml(string? auctionHtml)
{
if (string.IsNullOrEmpty(auctionHtml)) return null;
var content = auctionHtml ?? string.Empty;
var m = Regex.Match(content, "Vincitore[:\\s\\\"]+<[^>]*>(?<w>[^<]+)</", RegexOptions.IgnoreCase);
if (m.Success) return WebUtility.HtmlDecode(m.Groups["w"].Value).Trim();
m = Regex.Match(content, "<span[^>]*class=\\\"username\\\"[^>]*>.*?<span[^>]*class=\\\"offer\\\"[^>]*>(?<w>[^<]+)</span>", RegexOptions.IgnoreCase | RegexOptions.Singleline);
if (m.Success) return WebUtility.HtmlDecode(m.Groups["w"].Value).Trim();
m = Regex.Match(content, "mobile_offerer offer\\\"[^>]*>(?<w>[^<]+)<", RegexOptions.IgnoreCase);
if (m.Success) return WebUtility.HtmlDecode(m.Groups["w"].Value).Trim();
return null;
}
private int? ExtractBidsUsedFromAuctionHtml(string? auctionHtml)
{
if (string.IsNullOrEmpty(auctionHtml)) return null;
var content = auctionHtml ?? string.Empty;
// 1) Look for the explicit bids-used span: <p ...><span>628</span> Puntate utilizzate</p>
var m = Regex.Match(content, "class=\\\"bids-used\\\"[^>]*>[^<]*<span[^>]*>(?<n>[0-9]{1,7})</span>", RegexOptions.IgnoreCase | RegexOptions.Singleline);
if (m.Success && int.TryParse(m.Groups["n"].Value, out var val)) return val;
// 2) Look for numeric followed by 'Puntate utilizzate' or similar
m = Regex.Match(content, "(?<n>[0-9]{1,7})\\s*(?:Puntate utilizzate|Puntate usate|puntate utilizzate|puntate usate|puntate)\\b", RegexOptions.IgnoreCase);
if (m.Success && int.TryParse(m.Groups["n"].Value, out val)) return val;
// 3) Fallbacks used previously
m = Regex.Match(content, "(?<n>[0-9]+)\\s*(?:puntate|Puntate|puntate usate|puntate_usate|pt\\.?|pts)\\b", RegexOptions.IgnoreCase);
if (m.Success && int.TryParse(m.Groups["n"].Value, out val)) return val;
m = Regex.Match(content, "usato[sx]?\\s*(?<n>[0-9]{1,6})\\s*(?:puntate|pts|pt)\\b", RegexOptions.IgnoreCase);
if (m.Success && int.TryParse(m.Groups["n"].Value, out val)) return val;
m = Regex.Match(content, "(Puntate\\s*(?:usate|vinte)?)[^0-9]{0,10}(?<n>[0-9]{1,6})", RegexOptions.IgnoreCase);
if (m.Success && int.TryParse(m.Groups["n"].Value, out val)) return val;
m = Regex.Match(content, "<[^>]*>\\s*(?<n>[0-9]{1,6})\\s*(puntate)\\s*<", RegexOptions.IgnoreCase);
if (m.Success && int.TryParse(m.Groups["n"].Value, out val)) return val;
return null;
}
private double? ParseEuro(string s)
{
if (string.IsNullOrWhiteSpace(s)) return null;
s = s.Trim();
s = s.Replace(".", "").Replace(',', '.');
if (double.TryParse(s, NumberStyles.Any, CultureInfo.InvariantCulture, out var d)) return d;
return null;
}
private string StripTags(string input)
{
return Regex.Replace(input ?? string.Empty, "<.*?>", string.Empty);
}
private void SaveCsv(IEnumerable<Models.ClosedAuctionRecord> data, string filePath)
{
var sb = new StringBuilder();
sb.AppendLine("AuctionUrl,ProductName,FinalPrice,Winner,BidsUsed,ScrapedAt,Notes");
foreach (var r in data)
{
// Escape quotes
string Escape(string? v) => (v ?? string.Empty).Replace("\"", "\"\"");
var finalPrice = r.FinalPrice.HasValue ? r.FinalPrice.Value.ToString("F2", CultureInfo.InvariantCulture) : string.Empty;
var bidsUsed = r.BidsUsed.HasValue ? r.BidsUsed.Value.ToString() : string.Empty;
var line = $"\"{Escape(r.AuctionUrl)}\",\"{Escape(r.ProductName)}\",{finalPrice},\"{Escape(r.Winner)}\",{bidsUsed},\"{r.ScrapedAt:O}\",\"{Escape(r.Notes)}\"";
sb.AppendLine(line);
}
File.WriteAllText(filePath, sb.ToString(), Encoding.UTF8);
}
}
}
+109
View File
@@ -0,0 +1,109 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using AutoBidder.Data;
using AutoBidder.Models;
using System.Collections.Generic;
namespace AutoBidder.Services
{
public class StatsService
{
private readonly StatisticsContext _ctx;
public StatsService(StatisticsContext ctx)
{
_ctx = ctx;
// Ensure DB created
_ctx.Database.Migrate();
}
private static string NormalizeKey(string? productName, string? auctionUrl)
{
// Prefer auctionUrl numeric id if present
try
{
if (!string.IsNullOrWhiteSpace(auctionUrl))
{
var uri = new Uri(auctionUrl);
// Try regex to find trailing numeric ID
var m = System.Text.RegularExpressions.Regex.Match(uri.AbsolutePath + uri.Query, @"(\d{6,})");
if (m.Success) return m.Groups[1].Value;
}
}
catch { }
if (!string.IsNullOrWhiteSpace(productName))
{
var key = productName.Trim().ToLowerInvariant();
key = System.Text.RegularExpressions.Regex.Replace(key, "[^a-z0-9]+", "_");
return key;
}
return string.Empty;
}
public async Task RecordClosedAuctionAsync(ClosedAuctionRecord rec)
{
if (rec == null) return;
var key = NormalizeKey(rec.ProductName, rec.AuctionUrl);
if (string.IsNullOrWhiteSpace(key)) return;
var stat = await _ctx.ProductStats.FirstOrDefaultAsync(p => p.ProductKey == key);
if (stat == null)
{
stat = new ProductStat
{
ProductKey = key,
ProductName = rec.ProductName ?? "",
TotalAuctions = 0,
TotalBidsUsed = 0,
TotalFinalPriceCents = 0,
LastSeen = DateTime.UtcNow
};
_ctx.ProductStats.Add(stat);
}
stat.TotalAuctions += 1;
stat.TotalBidsUsed += rec.BidsUsed ?? 0;
if (rec.FinalPrice.HasValue)
stat.TotalFinalPriceCents += (long)Math.Round(rec.FinalPrice.Value * 100.0);
stat.LastSeen = DateTime.UtcNow;
await _ctx.SaveChangesAsync();
}
public async Task<ProductStat?> GetStatsForKeyAsync(string productName, string auctionUrl)
{
var key = NormalizeKey(productName, auctionUrl);
if (string.IsNullOrWhiteSpace(key)) return null;
return await _ctx.ProductStats.FirstOrDefaultAsync(p => p.ProductKey == key);
}
public async Task<(int recommendedBids, double recommendedMaxPrice)> GetRecommendationAsync(string productName, string auctionUrl, double userRiskFactor = 1.0)
{
var stat = await GetStatsForKeyAsync(productName, auctionUrl);
if (stat == null || stat.TotalAuctions < 3)
{
return (1, 1.0); // conservative defaults
}
int recBids = (int)Math.Ceiling(stat.AverageBidsUsed * userRiskFactor);
if (recBids < 1) recBids = 1;
// recommended max price: avg * (1 + min(0.2, 1/sqrt(n)))
double factor = 1.0 + Math.Min(0.2, 1.0 / Math.Sqrt(Math.Max(1, stat.TotalAuctions)));
double recPrice = stat.AverageFinalPrice * factor;
return (recBids, recPrice);
}
// New: return all stats for export
public async Task<List<ProductStat>> GetAllStatsAsync()
{
return await _ctx.ProductStats
.OrderByDescending(p => p.LastSeen)
.ToListAsync();
}
}
}