using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; using Blog3000.Shared; using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; using System.IO; using Blog3000.Server.MiddleWares; using System.Text.Json; using Microsoft.CodeAnalysis.CSharp; namespace Blog3000.Server { public class BlogPostRepo { public event EventHandler BlogPostsChanged; private static object updateLock = new object(); private static object idLock = new object(); private static long lastUsedTs; private static int CACHETIMEOUTMINS = 5; private static string BLOGPOSTKEY = "blog-posts-cache-key"; private static string BLOGPOSTUPDTIMEKEY = "blog-posts-upd-time-keycache-key"; private bool forceCacheUpdate = false; private Dictionary checksumsById = null; private readonly string _blogPostPath; private readonly string checksumCacheFile; private readonly ILogger logger; private readonly IMemoryCache cache; public BlogPostRepo(ILogger logger, BlogEnv config, IMemoryCache cache) { this.logger = logger; this.cache = cache; this._blogPostPath = System.IO.Path.Combine( //(string)AppDomain.CurrentDomain.GetData("AppDataPath"), config.AppDataPath, "posts"); this.checksumCacheFile = System.IO.Path.Combine( //(string)AppDomain.CurrentDomain.GetData("AppDataPath"), config.AppDataPath, "checksumcache.json"); UpdateCache(); } public string BlogPostRoot { get { return _blogPostPath; } } /// /// CreateId based on timestamp + author. /// - Time no allowed to go backwards, especially when Service not running /// - Not multiinstance/multihosts aware /// public static void CreateId(BlogPostHeader b) { if (String.IsNullOrEmpty(b.Id) && !String.IsNullOrEmpty(b.Author)) { lock (idLock) { var span = DateTime.UtcNow - new DateTime(2001, 1, 1); long ts = (long)(span.TotalMilliseconds / 10); // 1/10ms resolution if (ts <= lastUsedTs) // Time went backwards { ts = lastUsedTs + 1; } lastUsedTs = ts; b.Id = CreateId(b.Author, ts); System.Threading.Thread.Sleep(10); } } else { throw new InvalidOperationException("Id already set or author missing"); } } public void ClearCache() { forceCacheUpdate = true; } /// /// /// /// cached list of posts public List GetHeaders() { List res = GetHeaderFromCache(); if (res == null) { res = UpdateCache(); } return res; } public string GetSafeFilePath(BlogPostHeader b) { return GetSafeFilePath(b.Author, b.Filename); } public DateTime? GetFileModDate(BlogPostHeader b) { try { var p = GetSafeFilePath(b); var res = System.IO.File.GetLastWriteTimeUtc(p); return res; } catch (IOException ioex) { logger.LogWarning("Error gettime writetime of file", ioex); } return null; } public string GetText(BlogPostHeader b) { var p = GetSafeFilePath(b); var sb = new StringBuilder(); bool skipMode = true; try { foreach (var s in System.IO.File.ReadAllLines(p)) { if (skipMode && !s.StartsWith("@")) skipMode = false; if (!skipMode) { sb.AppendLine(s); } } } catch (System.IO.IOException fex) { logger.LogWarning("Error reading file", fex); return null; } return sb .Replace("$PUBLICDL$", "PublicDl") .ToString(); } /// /// Images for a given blogfn.md must reside in folder "blogfn-without-ext/fn" /// /// /// /// public string GetImgPath(BlogPostHeader b, string imgRef) { string imgRelPath = null; if (b.ImageRefs == null || !b.ImageRefs.TryGetValue(imgRef, out imgRelPath)) { return null; } var p = GetSafeImgFilePath(b.Author, b.Filename, System.IO.Path.GetFileName(imgRelPath)); try { //var bytes = System.IO.File.ReadAllBytes(p); //return bytes; if (System.IO.File.Exists(p)) { return p; } } catch (System.IO.IOException fex) { logger.LogWarning("Error reading file", fex); } return null; } private BlogPostHeader FromFile(string f) { BlogPostHeader res = null; if (f != null) { bool isDraft = false; using (var stream = System.IO.File.OpenText(f)) { string s = null; while ((s = stream.ReadLine()) != null) { if (s.StartsWith("@")) // Meta-headers without parameter { var parts = s.Split(':', 2); if (parts.Length == 1) { switch (parts[0]) { case "@Draft": isDraft = true; break; case "@NoSocialMediaInfo": res.NoSocialMediaInfo = true; break; } } else if (parts.Length == 2) // Meta-Headers with parameters { if (res == null) res = new BlogPostHeader(); res.Filename = System.IO.Path.GetFileName(f); switch (parts[0]) { case "@Draft": isDraft = true; break; case "@Id": res.Id = parts[1]; break; case "@Lang": var rr = parts[1]; if (!String.IsNullOrEmpty(rr)) { res.Lang = rr.ToLowerInvariant(); } break; case "@Title": res.Title = parts[1]; break; case "@Author": res.Author = parts[1]; break; case "@Abstract": res.Abstract = parts[1]; break; case "@Access": res.Access = parts[1]; break; case "@StickyMenuPos": int pos; if (Int32.TryParse(parts[1], out pos)) { res.StickyMenuPos = pos; } break; case "@Topics": res.Topics = new List((parts[1] ?? "").Split("|")); break; case "@Revision": var p = (parts[1] ?? "").Split("|"); try { Revision r = new Revision(); if (p.Length >= 1) { DateTime dt; if (DateTime.TryParse(p[0], out dt)) { r.ChangedAt = dt; } } r.Description = p.Length > 1 ? p[1] : null; r.Author = p.Length > 2 ? p[2] : null; res.Revisions ??= new List(); res.Revisions.Add(r); } catch (Exception) { logger.LogWarning($"Error parsing Revision on file {f}"); } break; } } } else if (s.StartsWith("[")) // ImageRefs { var match = Regex.Match(s, @"\[(.+)\]:(.+)"); if (match.Success && match.Groups.Count > 2) { res.ImageRefs ??= new Dictionary(); res.ImageRefs.Add(match.Groups[1].Value.Trim(), match.Groups[2].Value.Trim()); } // } else if (!String.IsNullOrWhiteSpace(s)) // Non-empty string { // End of header+imagerefs break; } } } if (!isDraft) { bool rewriteRevisions = false; if ((res.Revisions?.Count ?? 0) == 0) { res.Revisions ??= new List(); var r = new Revision(); r.Description = "Initial"; res.Revisions.Add(r); } foreach (var r in res.Revisions) { if (String.IsNullOrWhiteSpace(r.Author) && (!String.IsNullOrWhiteSpace(res.Author))) { r.Author = res.Author; rewriteRevisions = true; } if (r.ChangedAt == null) { r.ChangedAt ??= DateTime.UtcNow; rewriteRevisions = true; } } bool rewriteId = false; if (String.IsNullOrEmpty(res.Id) && !String.IsNullOrWhiteSpace(res.Author)) { CreateId(res); rewriteId = true; } if (rewriteId || rewriteRevisions) { var lines = new List(System.IO.File.ReadAllLines(f)); var outlines = new List(); outlines.Add($"@Id:{res.Id}"); foreach (var r in res.Revisions) { outlines.Add($"@Revision:{r.ChangedAt}|{r.Description}|{r.Author}"); } bool outOfHeader = false; foreach (var ln in lines) { if (!outOfHeader) { if (ln.StartsWith("@Id") || (ln.StartsWith("@Revision"))) continue; if (!ln.StartsWith("@")) outOfHeader = true; } outlines.Add(ln); } System.IO.File.WriteAllLines(f, outlines, Encoding.UTF8); } } if (!isDraft && BlogPostHeader.IsUsable(res)) { res.Checksum = CalcCheckSum(res); } else { res = null; } } return res; } #region Caching stuff /// /// Returns content of cache, updates cache if required /// private List GetHeaderFromCache() { List res = null; if (forceCacheUpdate) { lock(updateLock) { if (forceCacheUpdate) // Cache still needs update? { UpdateCache(); } } } if (cache.TryGetValue(BLOGPOSTKEY, out res)) { if (IsCacheExpired()) { lock (updateLock) { // Recheck if cache is still expired after we acquired lock if (IsCacheExpired()) { res = UpdateCache(); } } } } return res; } private bool IsCacheExpired() { DateTime dt; if (cache.TryGetValue(BLOGPOSTUPDTIMEKEY, out dt)) { if (dt != null && dt.AddMinutes(CACHETIMEOUTMINS).CompareTo(DateTime.UtcNow) < 0) { return true; } } return false; } /// /// /// /// The new data internal List UpdateCache() { lock (updateLock) { logger.LogWarning($"Updating BlogPostCache"); var res = new List(); foreach (var s in System.IO.Directory.GetFiles( BlogPostRoot, "*.md", System.IO.SearchOption.AllDirectories) ) { var be = FromFile(s); if (be != null) { res.Add(be); } else { logger.LogWarning($"Post {s} not usable"); } } cache.Set(BLOGPOSTKEY, res); cache.Set(BLOGPOSTUPDTIMEKEY, DateTime.UtcNow); forceCacheUpdate = false; // Compare with cached checksum for changes bool hasChanges = false; bool newChache = false; if (checksumsById == null) { checksumsById = LoadCheckSumsById(); if (checksumsById == null) { checksumsById = new Dictionary(); newChache = true; } } var dcCopy = new Dictionary(checksumsById ?? new Dictionary()); foreach (var bp in res) { string cs = null; if (dcCopy.TryGetValue(bp.Id, out cs)) { if (!String.Equals(cs, bp.Checksum)) { hasChanges = true; break; } dcCopy.Remove(bp.Id); } else { hasChanges = true; break; } } if (dcCopy.Count > 0) { hasChanges = true; } // Write new id/checksum dict var dc = new Dictionary(); foreach (var bp in res) { dc.Add(bp.Id, bp.Checksum); } checksumsById = dc; SaveCheckSumsById(checksumsById); // Notify listenera about changes if (hasChanges) { if (newChache) { logger.LogWarning("BlogPosts changed, but new or bad cache, not notifying listeners"); } else { logger.LogInformation("BlogPosts changed, notifying listeners"); BlogPostsChanged?.Invoke(this, EventArgs.Empty); } } else { logger.LogInformation("BlogPosts not changed, not notifying listeners"); } return res; } } #endregion #region File+Path utils private string GetSafeFilePath(string author, string filename) { var at = CheckPathPart(author); var fn = CheckPathPart(filename); return System.IO.Path.Combine(BlogPostRoot, at, fn); } private string GetSafeImgFilePath(string author, string blogFilenmame, string imgFilename) { var at = CheckPathPart(author); var bt = System.IO.Path.GetFileNameWithoutExtension(CheckPathPart(blogFilenmame)); var it = CheckPathPart(imgFilename); return System.IO.Path.Combine(BlogPostRoot, at, bt, it); } private static string CheckPathPart(string part) { if (String.IsNullOrWhiteSpace(part)) { throw new ArgumentException($"null or empty?"); } var pt = System.IO.Path.GetFileName(part.Trim()); if (!String.Equals(part, pt)) { throw new ArgumentException($"contains invalid path-information?"); } return pt; } #endregion private string CalcCheckSum(BlogPostHeader b) { StringBuilder sb = new StringBuilder(); sb.Append(b.Id ?? ""); sb.Append("|"); sb.Append(b.Filename ?? ""); sb.Append("|"); //sb.Append(res.Title ?? ""); //sb.Append("|"); //sb.Append(res.Author ?? ""); //sb.Append("|"); //sb.Append(res.Access ?? ""); //sb.Append("|"); //res.Topics.ForEach(t => sb.Append(t + ",")); //sb.Append(res.Revisions?.Count ?? 0); //sb.Append("|"); //sb.Append(res.Abstract ?? ""); //sb.Append("|"); //sb.Append(res.StickyMenuPos ?? -1); //sb.Append("|"); if (b.ImageRefs != null) { foreach (var fn in b.ImageRefs.Values.OrderBy(s => s)) { var fno = System.IO.Path.GetFileName(fn); var rfp = GetSafeImgFilePath(b.Author, b.Filename, fno); var bytes = new byte[] { }; try { bytes = System.IO.File.ReadAllBytes(rfp); } catch (System.IO.IOException ex) { logger.LogWarning($"ImgFile {rfp} not readable"); } var sha = System.Convert.ToBase64String( System.Security.Cryptography.SHA256.Create().ComputeHash(bytes) ); sb.Append(sha); sb.Append(";"); } } var f = GetSafeFilePath(b.Author, b.Filename); sb.Append(System.IO.File.ReadAllText(f)); return System.Convert.ToBase64String( System.Security.Cryptography.SHA256.Create().ComputeHash( System.Text.Encoding.UTF8.GetBytes(sb.ToString()) ) ); ; } private Dictionary LoadCheckSumsById() { Dictionary res = null; try { if (System.IO.File.Exists(checksumCacheFile)) { var text = System.IO.File.ReadAllText(checksumCacheFile); res = JsonSerializer.Deserialize>(text); } } catch (Exception ex) { logger.LogWarning(ex, $"Error reading checksumcache file ${checksumCacheFile}"); } return res; } private bool SaveCheckSumsById(Dictionary cbcById) { cbcById ??= new Dictionary(); try { var text = JsonSerializer.Serialize(cbcById); System.IO.File.WriteAllText(checksumCacheFile, text); return true; } catch (Exception ex) { logger.LogWarning(ex, $"Error writing checksumCache file ${checksumCacheFile}"); } return false; } private static string CreateId(string author, long ts) { const string radixChars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefhijklmnopqrstuvwxyz"; string res = ""; void radixer(long l, ref string res) { do { var r = l % radixChars.Length; res = radixChars[(int)r] + res; l = (l - r) / radixChars.Length; } while (l > 0); } radixer(ts, ref res); var authorPrefixBytes = System.Text.Encoding.UTF8.GetBytes(author); foreach (var b in authorPrefixBytes) { radixer(b, ref res); } return res; } } }