Compare commits
2 Commits
2b9c374e8c
...
7e9c87de44
Author | SHA1 | Date | |
---|---|---|---|
7e9c87de44 | |||
fda4c01c22 |
|
@ -7,6 +7,4 @@ public interface IEssayScanService
|
|||
public Task<BlogContents> ScanContents();
|
||||
|
||||
public Task SaveBlogContent(BlogContent content, bool isDraft = true);
|
||||
|
||||
public Task<ImageScanResult> ScanImages();
|
||||
}
|
||||
|
|
21
YaeBlog/Commands/Binders/ImageCompressServiceBinder.cs
Normal file
|
@ -0,0 +1,21 @@
|
|||
using System.CommandLine.Binding;
|
||||
using YaeBlog.Abstraction;
|
||||
using YaeBlog.Services;
|
||||
|
||||
namespace YaeBlog.Commands.Binders;
|
||||
|
||||
public sealed class ImageCompressServiceBinder : BinderBase<ImageCompressService>
|
||||
{
|
||||
protected override ImageCompressService GetBoundValue(BindingContext bindingContext)
|
||||
{
|
||||
bindingContext.AddService(provider =>
|
||||
{
|
||||
IEssayScanService essayScanService = provider.GetRequiredService<IEssayScanService>();
|
||||
ILogger<ImageCompressService> logger = provider.GetRequiredService<ILogger<ImageCompressService>>();
|
||||
|
||||
return new ImageCompressService(essayScanService, logger);
|
||||
});
|
||||
|
||||
return bindingContext.GetRequiredService<ImageCompressService>();
|
||||
}
|
||||
}
|
|
@ -1,4 +1,6 @@
|
|||
using System.CommandLine;
|
||||
using Microsoft.Extensions.Options;
|
||||
using YaeBlog.Abstraction;
|
||||
using YaeBlog.Commands.Binders;
|
||||
using YaeBlog.Components;
|
||||
using YaeBlog.Extensions;
|
||||
|
@ -19,6 +21,7 @@ public sealed class YaeBlogCommand
|
|||
AddNewCommand(_rootCommand);
|
||||
AddPublishCommand(_rootCommand);
|
||||
AddScanCommand(_rootCommand);
|
||||
AddCompressCommand(_rootCommand);
|
||||
}
|
||||
|
||||
public Task<int> RunAsync(string[] args)
|
||||
|
@ -94,22 +97,20 @@ public sealed class YaeBlogCommand
|
|||
Argument<string> filenameArgument = new(name: "blog name", description: "The created blog filename.");
|
||||
newCommand.AddArgument(filenameArgument);
|
||||
|
||||
newCommand.SetHandler(async (file, _, _, essayScanService) =>
|
||||
newCommand.SetHandler(async (file, blogOption, _, essayScanService) =>
|
||||
{
|
||||
BlogContents contents = await essayScanService.ScanContents();
|
||||
|
||||
if (contents.Posts.Any(content => content.FileName == file))
|
||||
if (contents.Posts.Any(content => content.BlogName == file))
|
||||
{
|
||||
Console.WriteLine("There exists the same title blog in posts.");
|
||||
return;
|
||||
}
|
||||
|
||||
await essayScanService.SaveBlogContent(new BlogContent
|
||||
{
|
||||
FileName = file,
|
||||
FileContent = string.Empty,
|
||||
Metadata = new MarkdownMetadata { Title = file, Date = DateTime.Now }
|
||||
});
|
||||
await essayScanService.SaveBlogContent(new BlogContent(
|
||||
new FileInfo(Path.Combine(blogOption.Value.Root, "drafts", file + ".md")),
|
||||
new MarkdownMetadata { Title = file, Date = DateTime.Now },
|
||||
string.Empty, true, [], []));
|
||||
|
||||
Console.WriteLine($"Created new blog '{file}.");
|
||||
}, filenameArgument, new BlogOptionsBinder(), new LoggerBinder<EssayScanService>(),
|
||||
|
@ -126,15 +127,15 @@ public sealed class YaeBlogCommand
|
|||
BlogContents contents = await essyScanService.ScanContents();
|
||||
|
||||
Console.WriteLine($"All {contents.Posts.Count} Posts:");
|
||||
foreach (BlogContent content in contents.Posts.OrderBy(x => x.FileName))
|
||||
foreach (BlogContent content in contents.Posts.OrderBy(x => x.BlogName))
|
||||
{
|
||||
Console.WriteLine($" - {content.FileName}");
|
||||
Console.WriteLine($" - {content.BlogName}");
|
||||
}
|
||||
|
||||
Console.WriteLine($"All {contents.Drafts.Count} Drafts:");
|
||||
foreach (BlogContent content in contents.Drafts.OrderBy(x => x.FileName))
|
||||
foreach (BlogContent content in contents.Drafts.OrderBy(x => x.BlogName))
|
||||
{
|
||||
Console.WriteLine($" - {content.FileName}");
|
||||
Console.WriteLine($" - {content.BlogName}");
|
||||
}
|
||||
}, new BlogOptionsBinder(), new LoggerBinder<EssayScanService>(), new EssayScanServiceBinder());
|
||||
}
|
||||
|
@ -150,32 +151,39 @@ public sealed class YaeBlogCommand
|
|||
|
||||
command.SetHandler(async (_, _, essayScanService, removeOptionValue) =>
|
||||
{
|
||||
ImageScanResult result = await essayScanService.ScanImages();
|
||||
BlogContents contents = await essayScanService.ScanContents();
|
||||
List<BlogImageInfo> unusedImages = (from content in contents
|
||||
from image in content.Images
|
||||
where image is { IsUsed: false }
|
||||
select image).ToList();
|
||||
|
||||
if (result.UnusedImages.Count != 0)
|
||||
if (unusedImages.Count != 0)
|
||||
{
|
||||
Console.WriteLine("Found unused images:");
|
||||
Console.WriteLine("HINT: use '--rm' to remove unused images.");
|
||||
}
|
||||
|
||||
foreach (FileInfo image in result.UnusedImages)
|
||||
foreach (BlogImageInfo image in unusedImages)
|
||||
{
|
||||
Console.WriteLine($" - {image.FullName}");
|
||||
Console.WriteLine($" - {image.File.FullName}");
|
||||
}
|
||||
|
||||
if (removeOptionValue)
|
||||
{
|
||||
foreach (FileInfo image in result.UnusedImages)
|
||||
foreach (BlogImageInfo image in unusedImages)
|
||||
{
|
||||
image.Delete();
|
||||
image.File.Delete();
|
||||
}
|
||||
}
|
||||
|
||||
Console.WriteLine("Used not existed images:");
|
||||
|
||||
foreach (FileInfo image in result.NotFoundImages)
|
||||
foreach (BlogContent content in contents)
|
||||
{
|
||||
Console.WriteLine($" - {image.FullName}");
|
||||
foreach (FileInfo file in content.NotfoundImages)
|
||||
{
|
||||
Console.WriteLine($"- {file.Name} in {content.BlogName}");
|
||||
}
|
||||
}
|
||||
}, new BlogOptionsBinder(), new LoggerBinder<EssayScanService>(), new EssayScanServiceBinder(), removeOption);
|
||||
}
|
||||
|
@ -193,7 +201,7 @@ public sealed class YaeBlogCommand
|
|||
BlogContents contents = await essayScanService.ScanContents();
|
||||
|
||||
BlogContent? content = (from blog in contents.Drafts
|
||||
where blog.FileName == filename
|
||||
where blog.BlogName == filename
|
||||
select blog).FirstOrDefault();
|
||||
|
||||
if (content is null)
|
||||
|
@ -202,14 +210,17 @@ public sealed class YaeBlogCommand
|
|||
return;
|
||||
}
|
||||
|
||||
// 设置发布的时间
|
||||
content.Metadata.Date = DateTime.Now;
|
||||
|
||||
// 将选中的博客文件复制到posts
|
||||
await essayScanService.SaveBlogContent(content, isDraft: false);
|
||||
|
||||
// 复制图片文件夹
|
||||
DirectoryInfo sourceImageDirectory =
|
||||
new(Path.Combine(blogOptions.Value.Root, "drafts", content.FileName));
|
||||
new(Path.Combine(blogOptions.Value.Root, "drafts", content.BlogName));
|
||||
DirectoryInfo targetImageDirectory =
|
||||
new(Path.Combine(blogOptions.Value.Root, "posts", content.FileName));
|
||||
new(Path.Combine(blogOptions.Value.Root, "posts", content.BlogName));
|
||||
|
||||
if (sourceImageDirectory.Exists)
|
||||
{
|
||||
|
@ -223,9 +234,30 @@ public sealed class YaeBlogCommand
|
|||
}
|
||||
|
||||
// 删除原始的文件
|
||||
FileInfo sourceBlogFile = new(Path.Combine(blogOptions.Value.Root, "drafts", content.FileName + ".md"));
|
||||
FileInfo sourceBlogFile = new(Path.Combine(blogOptions.Value.Root, "drafts", content.BlogName + ".md"));
|
||||
sourceBlogFile.Delete();
|
||||
}, new BlogOptionsBinder(),
|
||||
new LoggerBinder<EssayScanService>(), new EssayScanServiceBinder(), filenameArgument);
|
||||
}
|
||||
|
||||
private static void AddCompressCommand(RootCommand rootCommand)
|
||||
{
|
||||
Command command = new("compress", "Compress png/jpeg image to webp image to reduce size.");
|
||||
rootCommand.Add(command);
|
||||
|
||||
Option<bool> dryRunOption = new("--dry-run", description: "Dry run the compression task but not write.",
|
||||
getDefaultValue: () => false);
|
||||
command.AddOption(dryRunOption);
|
||||
|
||||
command.SetHandler(ImageCommandHandler,
|
||||
new BlogOptionsBinder(), new LoggerBinder<EssayScanService>(), new LoggerBinder<ImageCompressService>(),
|
||||
new EssayScanServiceBinder(), new ImageCompressServiceBinder(), dryRunOption);
|
||||
}
|
||||
|
||||
private static async Task ImageCommandHandler(IOptions<BlogOptions> _, ILogger<EssayScanService> _1,
|
||||
ILogger<ImageCompressService> _2,
|
||||
IEssayScanService _3, ImageCompressService imageCompressService, bool dryRun)
|
||||
{
|
||||
await imageCompressService.Compress(dryRun);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,12 +1,20 @@
|
|||
namespace YaeBlog.Models;
|
||||
|
||||
public class BlogContent
|
||||
/// <summary>
|
||||
/// 单个博客文件的所有数据和元数据
|
||||
/// </summary>
|
||||
/// <param name="BlogFile">博客文件</param>
|
||||
/// <param name="Metadata">文件中的MD元数据</param>
|
||||
/// <param name="Content">文件内容</param>
|
||||
/// <param name="IsDraft">是否为草稿</param>
|
||||
/// <param name="Images">博客中使用的文件</param>
|
||||
public record BlogContent(
|
||||
FileInfo BlogFile,
|
||||
MarkdownMetadata Metadata,
|
||||
string Content,
|
||||
bool IsDraft,
|
||||
List<BlogImageInfo> Images,
|
||||
List<FileInfo> NotfoundImages)
|
||||
{
|
||||
public required string FileName { get; init; }
|
||||
|
||||
public required MarkdownMetadata Metadata { get; init; }
|
||||
|
||||
public required string FileContent { get; set; }
|
||||
|
||||
public bool IsDraft { get; set; } = false;
|
||||
public string BlogName => BlogFile.Name.Split('.')[0];
|
||||
}
|
||||
|
|
|
@ -1,10 +1,15 @@
|
|||
using System.Collections.Concurrent;
|
||||
using System.Collections;
|
||||
using System.Collections.Concurrent;
|
||||
|
||||
namespace YaeBlog.Models;
|
||||
|
||||
public sealed class BlogContents(ConcurrentBag<BlogContent> drafts, ConcurrentBag<BlogContent> posts)
|
||||
public record BlogContents(ConcurrentBag<BlogContent> Drafts, ConcurrentBag<BlogContent> Posts)
|
||||
: IEnumerable<BlogContent>
|
||||
{
|
||||
public ConcurrentBag<BlogContent> Drafts { get; } = drafts;
|
||||
IEnumerator<BlogContent> IEnumerable<BlogContent>.GetEnumerator()
|
||||
{
|
||||
return Posts.Concat(Drafts).GetEnumerator();
|
||||
}
|
||||
|
||||
public ConcurrentBag<BlogContent> Posts { get; } = posts;
|
||||
public IEnumerator GetEnumerator() => ((IEnumerable<BlogContent>)this).GetEnumerator();
|
||||
}
|
||||
|
|
44
YaeBlog/Models/BlogImageInfo.cs
Normal file
|
@ -0,0 +1,44 @@
|
|||
using System.Text;
|
||||
|
||||
namespace YaeBlog.Models;
|
||||
|
||||
public record BlogImageInfo(FileInfo File, long Width, long Height, string MineType, byte[] Content, bool IsUsed)
|
||||
: IComparable<BlogImageInfo>
|
||||
{
|
||||
public int Size => Content.Length;
|
||||
|
||||
public override string ToString()
|
||||
{
|
||||
StringBuilder builder = new();
|
||||
|
||||
builder.AppendLine($"Blog image {File.Name}:");
|
||||
builder.AppendLine($"\tWidth: {Width}; Height: {Height}");
|
||||
builder.AppendLine($"\tSize: {FormatSize()}");
|
||||
builder.AppendLine($"\tImage Format: {MineType}");
|
||||
|
||||
return builder.ToString();
|
||||
}
|
||||
|
||||
public int CompareTo(BlogImageInfo? other)
|
||||
{
|
||||
if (other is null)
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
|
||||
return other.Size.CompareTo(Size);
|
||||
}
|
||||
|
||||
private string FormatSize()
|
||||
{
|
||||
double size = Size;
|
||||
if (size / 1024 > 3)
|
||||
{
|
||||
size /= 1024;
|
||||
|
||||
return size / 1024 > 3 ? $"{size / 1024}MB" : $"{size}KB";
|
||||
}
|
||||
|
||||
return $"{size}B";
|
||||
}
|
||||
}
|
|
@ -1,3 +0,0 @@
|
|||
namespace YaeBlog.Models;
|
||||
|
||||
public record struct ImageScanResult(List<FileInfo> UnusedImages, List<FileInfo> NotFoundImages);
|
|
@ -1,5 +1,7 @@
|
|||
using System.Collections.Concurrent;
|
||||
using System.Text.RegularExpressions;
|
||||
using Imageflow.Bindings;
|
||||
using Imageflow.Fluent;
|
||||
using Microsoft.Extensions.Options;
|
||||
using YaeBlog.Abstraction;
|
||||
using YaeBlog.Core.Exceptions;
|
||||
|
@ -9,17 +11,30 @@ using YamlDotNet.Serialization;
|
|||
|
||||
namespace YaeBlog.Services;
|
||||
|
||||
public partial class EssayScanService(
|
||||
ISerializer yamlSerializer,
|
||||
IDeserializer yamlDeserializer,
|
||||
IOptions<BlogOptions> blogOptions,
|
||||
ILogger<EssayScanService> logger) : IEssayScanService
|
||||
public partial class EssayScanService : IEssayScanService
|
||||
{
|
||||
private readonly BlogOptions _blogOptions = blogOptions.Value;
|
||||
private readonly BlogOptions _blogOptions;
|
||||
private readonly ISerializer _yamlSerializer;
|
||||
private readonly IDeserializer _yamlDeserializer;
|
||||
private readonly ILogger<EssayScanService> _logger;
|
||||
|
||||
public EssayScanService(ISerializer yamlSerializer,
|
||||
IDeserializer yamlDeserializer,
|
||||
IOptions<BlogOptions> blogOptions,
|
||||
ILogger<EssayScanService> logger)
|
||||
{
|
||||
_yamlSerializer = yamlSerializer;
|
||||
_yamlDeserializer = yamlDeserializer;
|
||||
_logger = logger;
|
||||
_blogOptions = blogOptions.Value;
|
||||
RootDirectory = ValidateRootDirectory();
|
||||
}
|
||||
|
||||
private DirectoryInfo RootDirectory { get; }
|
||||
|
||||
public async Task<BlogContents> ScanContents()
|
||||
{
|
||||
ValidateDirectory(_blogOptions.Root, out DirectoryInfo drafts, out DirectoryInfo posts);
|
||||
ValidateDirectory(out DirectoryInfo drafts, out DirectoryInfo posts);
|
||||
|
||||
return new BlogContents(
|
||||
await ScanContentsInternal(drafts, true),
|
||||
|
@ -28,82 +43,92 @@ public partial class EssayScanService(
|
|||
|
||||
public async Task SaveBlogContent(BlogContent content, bool isDraft = true)
|
||||
{
|
||||
ValidateDirectory(_blogOptions.Root, out DirectoryInfo drafts, out DirectoryInfo posts);
|
||||
ValidateDirectory(out DirectoryInfo drafts, out DirectoryInfo posts);
|
||||
|
||||
FileInfo targetFile = isDraft
|
||||
? new FileInfo(Path.Combine(drafts.FullName, content.FileName + ".md"))
|
||||
: new FileInfo(Path.Combine(posts.FullName, content.FileName + ".md"));
|
||||
|
||||
if (!isDraft)
|
||||
{
|
||||
content.Metadata.Date = DateTime.Now;
|
||||
}
|
||||
? new FileInfo(Path.Combine(drafts.FullName, content.BlogName + ".md"))
|
||||
: new FileInfo(Path.Combine(posts.FullName, content.BlogName + ".md"));
|
||||
|
||||
if (targetFile.Exists)
|
||||
{
|
||||
logger.LogWarning("Blog {} exists, overriding.", targetFile.Name);
|
||||
_logger.LogWarning("Blog {} exists, overriding.", targetFile.Name);
|
||||
}
|
||||
|
||||
await using StreamWriter writer = targetFile.CreateText();
|
||||
|
||||
await writer.WriteAsync("---\n");
|
||||
await writer.WriteAsync(yamlSerializer.Serialize(content.Metadata));
|
||||
await writer.WriteAsync(_yamlSerializer.Serialize(content.Metadata));
|
||||
await writer.WriteAsync("---\n");
|
||||
|
||||
if (isDraft)
|
||||
if (string.IsNullOrEmpty(content.Content) && isDraft)
|
||||
{
|
||||
// 如果博客为操作且内容为空
|
||||
// 创建简介隔断符号
|
||||
await writer.WriteLineAsync("<!--more-->");
|
||||
}
|
||||
else
|
||||
{
|
||||
await writer.WriteAsync(content.FileContent);
|
||||
await writer.WriteAsync(content.Content);
|
||||
}
|
||||
|
||||
// 保存图片文件
|
||||
await Task.WhenAll(from image in content.Images
|
||||
select File.WriteAllBytesAsync(image.File.FullName, image.Content));
|
||||
}
|
||||
|
||||
private record struct BlogResult(
|
||||
FileInfo BlogFile,
|
||||
string BlogContent,
|
||||
List<BlogImageInfo> Images,
|
||||
List<FileInfo> NotFoundImages);
|
||||
|
||||
private async Task<ConcurrentBag<BlogContent>> ScanContentsInternal(DirectoryInfo directory, bool isDraft)
|
||||
{
|
||||
// 扫描以md结果的但是不是隐藏文件的文件
|
||||
// 扫描以md结尾且不是隐藏文件的文件
|
||||
IEnumerable<FileInfo> markdownFiles = from file in directory.EnumerateFiles()
|
||||
where file.Extension == ".md" && !file.Name.StartsWith('.')
|
||||
select file;
|
||||
|
||||
ConcurrentBag<(string, string)> fileContents = [];
|
||||
ConcurrentBag<BlogResult> fileContents = [];
|
||||
|
||||
await Parallel.ForEachAsync(markdownFiles, async (file, token) =>
|
||||
{
|
||||
using StreamReader reader = file.OpenText();
|
||||
fileContents.Add((file.Name, await reader.ReadToEndAsync(token)));
|
||||
string blogName = file.Name.Split('.')[0];
|
||||
string blogContent = await reader.ReadToEndAsync(token);
|
||||
ImageResult imageResult =
|
||||
await ScanImagePreBlog(directory, blogName,
|
||||
blogContent);
|
||||
|
||||
fileContents.Add(new BlogResult(file, blogContent, imageResult.Images, imageResult.NotfoundImages));
|
||||
});
|
||||
|
||||
ConcurrentBag<BlogContent> contents = [];
|
||||
|
||||
await Task.Run(() =>
|
||||
{
|
||||
foreach ((string filename, string content) in fileContents)
|
||||
foreach (BlogResult blog in fileContents)
|
||||
{
|
||||
int endPos = content.IndexOf("---", 4, StringComparison.Ordinal);
|
||||
if (!content.StartsWith("---") || endPos is -1 or 0)
|
||||
int endPos = blog.BlogContent.IndexOf("---", 4, StringComparison.Ordinal);
|
||||
if (!blog.BlogContent.StartsWith("---") || endPos is -1 or 0)
|
||||
{
|
||||
logger.LogWarning("Failed to parse metadata from {}, skipped.", filename);
|
||||
_logger.LogWarning("Failed to parse metadata from {}, skipped.", blog.BlogFile.Name);
|
||||
return;
|
||||
}
|
||||
|
||||
string metadataString = content[4..endPos];
|
||||
string metadataString = blog.BlogContent[4..endPos];
|
||||
|
||||
try
|
||||
{
|
||||
MarkdownMetadata metadata = yamlDeserializer.Deserialize<MarkdownMetadata>(metadataString);
|
||||
logger.LogDebug("Scan metadata title: '{}' for {}.", metadata.Title, filename);
|
||||
MarkdownMetadata metadata = _yamlDeserializer.Deserialize<MarkdownMetadata>(metadataString);
|
||||
_logger.LogDebug("Scan metadata title: '{}' for {}.", metadata.Title, blog.BlogFile.Name);
|
||||
|
||||
contents.Add(new BlogContent
|
||||
{
|
||||
FileName = filename[..^3], Metadata = metadata, FileContent = content[(endPos + 3)..],
|
||||
IsDraft = isDraft
|
||||
});
|
||||
contents.Add(new BlogContent(blog.BlogFile, metadata, blog.BlogContent[(endPos + 3)..], isDraft,
|
||||
blog.Images, blog.NotFoundImages));
|
||||
}
|
||||
catch (YamlException e)
|
||||
{
|
||||
logger.LogWarning("Failed to parser metadata from {} due to {}, skipping", filename, e);
|
||||
_logger.LogWarning("Failed to parser metadata from {} due to {}, skipping", blog.BlogFile.Name, e);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
@ -111,99 +136,96 @@ public partial class EssayScanService(
|
|||
return contents;
|
||||
}
|
||||
|
||||
public async Task<ImageScanResult> ScanImages()
|
||||
private record struct ImageResult(List<BlogImageInfo> Images, List<FileInfo> NotfoundImages);
|
||||
|
||||
private async Task<ImageResult> ScanImagePreBlog(DirectoryInfo directory, string blogName, string content)
|
||||
{
|
||||
BlogContents contents = await ScanContents();
|
||||
ValidateDirectory(_blogOptions.Root, out DirectoryInfo drafts, out DirectoryInfo posts);
|
||||
MatchCollection matchResult = ImagePattern.Matches(content);
|
||||
DirectoryInfo imageDirectory = new(Path.Combine(directory.FullName, blogName));
|
||||
|
||||
List<FileInfo> unusedFiles = [];
|
||||
List<FileInfo> notFoundFiles = [];
|
||||
Dictionary<string, bool> usedImages = imageDirectory.Exists
|
||||
? imageDirectory.EnumerateFiles().ToDictionary(file => file.FullName, _ => false)
|
||||
: [];
|
||||
List<FileInfo> notFoundImages = [];
|
||||
|
||||
ImageScanResult draftResult = await ScanUnusedImagesInternal(contents.Drafts, drafts);
|
||||
ImageScanResult postResult = await ScanUnusedImagesInternal(contents.Posts, posts);
|
||||
|
||||
unusedFiles.AddRange(draftResult.UnusedImages);
|
||||
notFoundFiles.AddRange(draftResult.NotFoundImages);
|
||||
unusedFiles.AddRange(postResult.UnusedImages);
|
||||
notFoundFiles.AddRange(postResult.NotFoundImages);
|
||||
|
||||
return new ImageScanResult(unusedFiles, notFoundFiles);
|
||||
}
|
||||
|
||||
private static Task<ImageScanResult> ScanUnusedImagesInternal(IEnumerable<BlogContent> contents,
|
||||
DirectoryInfo root)
|
||||
{
|
||||
ConcurrentBag<FileInfo> unusedImage = [];
|
||||
ConcurrentBag<FileInfo> notFoundImage = [];
|
||||
|
||||
Parallel.ForEach(contents, content =>
|
||||
foreach (Match match in matchResult)
|
||||
{
|
||||
MatchCollection result = ImagePattern.Matches(content.FileContent);
|
||||
DirectoryInfo imageDirectory = new(Path.Combine(root.FullName, content.FileName));
|
||||
string imageName = match.Groups[1].Value;
|
||||
|
||||
Dictionary<string, bool> usedDictionary;
|
||||
// 判断md文件中的图片名称中是否包含文件夹名称
|
||||
// 例如 blog-1/image.png 或者 image.png
|
||||
// 如果不带文件夹名称
|
||||
// 默认添加同博客名文件夹
|
||||
FileInfo usedFile = imageName.Contains(blogName)
|
||||
? new FileInfo(Path.Combine(directory.FullName, imageName))
|
||||
: new FileInfo(Path.Combine(directory.FullName, blogName, imageName));
|
||||
|
||||
if (imageDirectory.Exists)
|
||||
if (usedImages.TryGetValue(usedFile.FullName, out _))
|
||||
{
|
||||
usedDictionary = (from file in imageDirectory.EnumerateFiles()
|
||||
select new KeyValuePair<string, bool>(file.FullName, false)).ToDictionary();
|
||||
usedImages[usedFile.FullName] = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
usedDictionary = [];
|
||||
notFoundImages.Add(usedFile);
|
||||
}
|
||||
}
|
||||
|
||||
foreach (Match match in result)
|
||||
{
|
||||
string imageName = match.Groups[1].Value;
|
||||
List<BlogImageInfo> images = (await Task.WhenAll((from pair in usedImages
|
||||
select GetImageInfo(new FileInfo(pair.Key), pair.Value)).ToArray())).ToList();
|
||||
|
||||
FileInfo usedFile = imageName.Contains(content.FileName)
|
||||
? new FileInfo(Path.Combine(root.FullName, imageName))
|
||||
: new FileInfo(Path.Combine(root.FullName, content.FileName, imageName));
|
||||
return new ImageResult(images, notFoundImages);
|
||||
}
|
||||
|
||||
if (usedDictionary.TryGetValue(usedFile.FullName, out _))
|
||||
{
|
||||
usedDictionary[usedFile.FullName] = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
notFoundImage.Add(usedFile);
|
||||
}
|
||||
}
|
||||
private static async Task<BlogImageInfo> GetImageInfo(FileInfo file, bool isUsed)
|
||||
{
|
||||
byte[] image = await File.ReadAllBytesAsync(file.FullName);
|
||||
|
||||
foreach (KeyValuePair<string, bool> pair in usedDictionary.Where(p => !p.Value))
|
||||
{
|
||||
unusedImage.Add(new FileInfo(pair.Key));
|
||||
}
|
||||
});
|
||||
if (file.Extension is ".jpg" or ".jpeg" or ".png")
|
||||
{
|
||||
ImageInfo imageInfo =
|
||||
await ImageJob.GetImageInfoAsync(MemorySource.Borrow(image), SourceLifetime.NowOwnedAndDisposedByTask);
|
||||
|
||||
return Task.FromResult(new ImageScanResult(unusedImage.ToList(), notFoundImage.ToList()));
|
||||
return new BlogImageInfo(file, imageInfo.ImageWidth, imageInfo.ImageWidth, imageInfo.PreferredMimeType,
|
||||
image, isUsed);
|
||||
}
|
||||
|
||||
return new BlogImageInfo(file, 0, 0, file.Extension switch
|
||||
{
|
||||
"svg" => "image/svg",
|
||||
"avif" => "image/avif",
|
||||
_ => string.Empty
|
||||
}, image, isUsed);
|
||||
}
|
||||
|
||||
[GeneratedRegex(@"\!\[.*?\]\((.*?)\)")]
|
||||
private static partial Regex ImagePattern { get; }
|
||||
|
||||
private void ValidateDirectory(string root, out DirectoryInfo drafts, out DirectoryInfo posts)
|
||||
|
||||
private DirectoryInfo ValidateRootDirectory()
|
||||
{
|
||||
root = Path.Combine(Environment.CurrentDirectory, root);
|
||||
DirectoryInfo rootDirectory = new(root);
|
||||
DirectoryInfo rootDirectory = new(Path.Combine(Environment.CurrentDirectory, _blogOptions.Root));
|
||||
|
||||
if (!rootDirectory.Exists)
|
||||
{
|
||||
throw new BlogFileException($"'{root}' is not a directory.");
|
||||
throw new BlogFileException($"'{_blogOptions.Root}' is not a directory.");
|
||||
}
|
||||
|
||||
if (rootDirectory.EnumerateDirectories().All(dir => dir.Name != "drafts"))
|
||||
return rootDirectory;
|
||||
}
|
||||
|
||||
private void ValidateDirectory(out DirectoryInfo drafts, out DirectoryInfo posts)
|
||||
{
|
||||
if (RootDirectory.EnumerateDirectories().All(dir => dir.Name != "drafts"))
|
||||
{
|
||||
throw new BlogFileException($"'{root}/drafts' not exists.");
|
||||
throw new BlogFileException($"'{_blogOptions.Root}/drafts' not exists.");
|
||||
}
|
||||
|
||||
if (rootDirectory.EnumerateDirectories().All(dir => dir.Name != "posts"))
|
||||
if (RootDirectory.EnumerateDirectories().All(dir => dir.Name != "posts"))
|
||||
{
|
||||
throw new BlogFileException($"'{root}/posts' not exists.");
|
||||
throw new BlogFileException($"'{_blogOptions.Root}/posts' not exists.");
|
||||
}
|
||||
|
||||
drafts = new DirectoryInfo(Path.Combine(root, "drafts"));
|
||||
posts = new DirectoryInfo(Path.Combine(root, "posts"));
|
||||
drafts = new DirectoryInfo(Path.Combine(_blogOptions.Root, "drafts"));
|
||||
posts = new DirectoryInfo(Path.Combine(_blogOptions.Root, "posts"));
|
||||
}
|
||||
}
|
||||
|
|
108
YaeBlog/Services/ImageCompressService.cs
Normal file
|
@ -0,0 +1,108 @@
|
|||
using Imageflow.Fluent;
|
||||
using YaeBlog.Abstraction;
|
||||
using YaeBlog.Core.Exceptions;
|
||||
using YaeBlog.Models;
|
||||
|
||||
namespace YaeBlog.Services;
|
||||
|
||||
public sealed class ImageCompressService(IEssayScanService essayScanService, ILogger<ImageCompressService> logger)
|
||||
{
|
||||
private record struct CompressResult(BlogImageInfo ImageInfo, byte[] CompressContent);
|
||||
|
||||
public async Task<List<BlogImageInfo>> ScanUsedImages()
|
||||
{
|
||||
BlogContents contents = await essayScanService.ScanContents();
|
||||
List<BlogImageInfo> originalImages = (from content in contents.Posts.Concat(contents.Drafts)
|
||||
from image in content.Images
|
||||
where image.IsUsed
|
||||
select image).ToList();
|
||||
|
||||
originalImages.Sort();
|
||||
|
||||
return originalImages;
|
||||
}
|
||||
|
||||
public async Task Compress(bool dryRun)
|
||||
{
|
||||
BlogContents contents = await essayScanService.ScanContents();
|
||||
|
||||
// 筛选需要压缩的图片
|
||||
// 即图片被博客使用且是jpeg/png格式
|
||||
List<BlogContent> needCompressContents = (from content in contents
|
||||
where content.Images.Any(i => i is { IsUsed: true } and { File.Extension: ".jpg" or ".jpeg" or ".png" })
|
||||
select content).ToList();
|
||||
|
||||
if (needCompressContents.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
int uncompressedSize = 0;
|
||||
int compressedSize = 0;
|
||||
List<BlogContent> compressedContent = new(needCompressContents.Count);
|
||||
|
||||
foreach (BlogContent content in needCompressContents)
|
||||
{
|
||||
List<BlogImageInfo> uncompressedImages = (from image in content.Images
|
||||
where image is { IsUsed: true } and { File.Extension: ".jpg" or ".jpeg" or ".png" }
|
||||
select image).ToList();
|
||||
|
||||
uncompressedSize += uncompressedImages.Select(i => i.Size).Sum();
|
||||
|
||||
foreach (BlogImageInfo image in uncompressedImages)
|
||||
{
|
||||
logger.LogInformation("Uncompressed image: {} belonging to blog {}.", image.File.Name,
|
||||
content.BlogName);
|
||||
}
|
||||
|
||||
CompressResult[] compressedImages = (await Task.WhenAll(from image in uncompressedImages
|
||||
select Task.Run(async () => new CompressResult(image, await ConvertToWebp(image.Content))))).ToArray();
|
||||
|
||||
compressedSize += compressedImages.Select(i => i.CompressContent.Length).Sum();
|
||||
|
||||
// 直接在原有的图片列表上添加图片
|
||||
List<BlogImageInfo> images = content.Images.Concat(from r in compressedImages
|
||||
select r.ImageInfo with
|
||||
{
|
||||
File = new FileInfo(r.ImageInfo.File.FullName.Split('.')[0] + ".webp"),
|
||||
Content = r.CompressContent
|
||||
}).ToList();
|
||||
// 修改文本
|
||||
string blogContent = compressedImages.Aggregate(content.Content, (c, r) =>
|
||||
{
|
||||
string originalName = r.ImageInfo.File.Name;
|
||||
string outputName = originalName.Split('.')[0] + ".webp";
|
||||
|
||||
return c.Replace(originalName, outputName);
|
||||
});
|
||||
|
||||
compressedContent.Add(content with { Images = images, Content = blogContent });
|
||||
}
|
||||
|
||||
logger.LogInformation("Compression ratio: {}%.", (double)compressedSize / uncompressedSize * 100.0);
|
||||
|
||||
if (dryRun is false)
|
||||
{
|
||||
await Task.WhenAll(from content in compressedContent
|
||||
select essayScanService.SaveBlogContent(content, content.IsDraft));
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<byte[]> ConvertToWebp(byte[] image)
|
||||
{
|
||||
using ImageJob job = new();
|
||||
BuildJobResult result = await job.Decode(MemorySource.Borrow(image))
|
||||
.EncodeToBytes(new WebPLossyEncoder(75))
|
||||
.Finish()
|
||||
.InProcessAsync();
|
||||
|
||||
ArraySegment<byte>? array = result.First?.TryGetBytes();
|
||||
|
||||
if (array.HasValue)
|
||||
{
|
||||
return array.Value.ToArray();
|
||||
}
|
||||
|
||||
throw new BlogFileException();
|
||||
}
|
||||
}
|
|
@ -41,14 +41,14 @@ public partial class RendererService(
|
|||
uint wordCount = GetWordCount(content);
|
||||
BlogEssay essay = new()
|
||||
{
|
||||
Title = content.Metadata.Title ?? content.FileName,
|
||||
FileName = content.FileName,
|
||||
Title = content.Metadata.Title ?? content.BlogName,
|
||||
FileName = content.BlogName,
|
||||
IsDraft = content.IsDraft,
|
||||
Description = GetDescription(content),
|
||||
WordCount = wordCount,
|
||||
ReadTime = CalculateReadTime(wordCount),
|
||||
PublishTime = content.Metadata.Date ?? DateTime.Now,
|
||||
HtmlContent = content.FileContent
|
||||
HtmlContent = content.Content
|
||||
};
|
||||
|
||||
if (content.Metadata.Tags is not null)
|
||||
|
@ -156,17 +156,17 @@ public partial class RendererService(
|
|||
private string GetDescription(BlogContent content)
|
||||
{
|
||||
const string delimiter = "<!--more-->";
|
||||
int pos = content.FileContent.IndexOf(delimiter, StringComparison.Ordinal);
|
||||
int pos = content.Content.IndexOf(delimiter, StringComparison.Ordinal);
|
||||
bool breakSentence = false;
|
||||
|
||||
if (pos == -1)
|
||||
{
|
||||
// 自动截取前50个字符
|
||||
pos = content.FileContent.Length < 50 ? content.FileContent.Length : 50;
|
||||
pos = content.Content.Length < 50 ? content.Content.Length : 50;
|
||||
breakSentence = true;
|
||||
}
|
||||
|
||||
string rawContent = content.FileContent[..pos];
|
||||
string rawContent = content.Content[..pos];
|
||||
MatchCollection matches = DescriptionPattern.Matches(rawContent);
|
||||
|
||||
StringBuilder builder = new();
|
||||
|
@ -182,18 +182,18 @@ public partial class RendererService(
|
|||
|
||||
string description = builder.ToString();
|
||||
|
||||
logger.LogDebug("Description of {} is {}.", content.FileName,
|
||||
logger.LogDebug("Description of {} is {}.", content.BlogName,
|
||||
description);
|
||||
return description;
|
||||
}
|
||||
|
||||
private uint GetWordCount(BlogContent content)
|
||||
{
|
||||
int count = (from c in content.FileContent
|
||||
int count = (from c in content.Content
|
||||
where char.IsLetterOrDigit(c)
|
||||
select c).Count();
|
||||
|
||||
logger.LogDebug("Word count of {} is {}", content.FileName,
|
||||
logger.LogDebug("Word count of {} is {}", content.BlogName,
|
||||
count);
|
||||
return (uint)count;
|
||||
}
|
||||
|
|
|
@ -1,6 +1,8 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk.Web">
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="ImageFlow.NativeRuntime.ubuntu-x86_64" Version="2.1.0-rc11"/>
|
||||
<PackageReference Include="ImageFlow.Net" Version="0.13.2"/>
|
||||
<PackageReference Include="System.CommandLine" Version="2.0.0-beta4.22272.1"/>
|
||||
<PackageReference Include="AngleSharp" Version="1.1.0"/>
|
||||
<PackageReference Include="Markdig" Version="0.38.0"/>
|
||||
|
|
|
@ -23,7 +23,7 @@ tags:
|
|||
在前12年的学生生涯中,我们都在期待着这一次的暑假,以为在这个没有作业的假期里,我们就可以充分的享受人间的美好。可是,当时我们不知道,这人间的烦恼,可不止作业这一种,无论是突如其来的疫情导致开学延期,还是等待录取时的不安。
|
||||
虽说在暑假时,拥有了自己的笔记本电脑,可是在高中三年屯下的游戏还是没有玩几个,看来我也是“喜加一”的受害者。虽然在高考后入坑了原神,但是假期间我并没有太过投入的玩。
|
||||
暑假下定决心要好好的学一学,可是看着我gitee上暑假期间那稀疏的提交,我就知道我又摸了一个暑假的鱼。
|
||||

|
||||

|
||||
即使我想写的很多项目都没有被扎实的推进下来,但是学习的一些的C语言还是让我受益匪浅。
|
||||
现在看来,这个假期真是,**学也没有学好,耍也没有耍好**的典型。
|
||||
|
||||
|
|
BIN
YaeBlog/source/posts/2021-final/1.png
(Stored with Git LFS)
BIN
YaeBlog/source/posts/2021-final/1.webp
Normal file
After Width: | Height: | Size: 9.8 KiB |
|
@ -57,11 +57,11 @@ date: 2022-12-30 14:58:12
|
|||
|
||||
小小的总结一下:2022年可以算得上是一事无成的一年,还搞砸了不少的事情。在写代码上进展有限,成绩上大幅倒退,说好的六级英语和大学物理竞赛都没有参加,在年末应对疫情进展的时候更是把“不知所措”这个成语诠释的淋漓尽致。
|
||||
|
||||

|
||||

|
||||
|
||||
关于今年的人际交往和社会关系,我愿意用QQ2022年年终总结中的一张截屏来总结,这张图片透漏出一种无可救药的悲伤。
|
||||
|
||||

|
||||

|
||||
|
||||
## 展望
|
||||
|
||||
|
|
BIN
YaeBlog/source/posts/2022-final/2022-12-30-14-26-19-QQ_Image_1672381538441.jpg
(Stored with Git LFS)
After Width: | Height: | Size: 34 KiB |
BIN
YaeBlog/source/posts/2022-final/2022-12-30-14-28-12-QQ_Image_1672381543836.jpg
(Stored with Git LFS)
After Width: | Height: | Size: 54 KiB |
|
@ -32,7 +32,7 @@ date: 2022-08-22 15:39:13
|
|||
- 下定决定要参加下一学期的物理竞赛,但是在听了讲座之后直接决定开学再开始学习,~~我知道我在家没法学习,俗称开摆~~
|
||||
- 又捡起了`Blender`,并在[Github](https://github.com/tanjian1998/bupt_minecraft)上找到了伟大的前辈们在`Minecraft`里复刻的老校区,希望能用`Blender`渲染几张图当作桌面。
|
||||
|
||||

|
||||

|
||||
|
||||
> 在此感谢所有为此付出过汗水的前辈们,让我这个即将搬入老校区的萌新能提前一睹老校区的风采。
|
||||
|
||||
|
|
BIN
YaeBlog/source/posts/2022-summer-vacation/result1.png
(Stored with Git LFS)
BIN
YaeBlog/source/posts/2022-summer-vacation/result1.webp
Normal file
After Width: | Height: | Size: 134 KiB |
|
@ -44,7 +44,7 @@ date: 2024-2-29 20:18:19
|
|||
|
||||
2023年最令我吃惊的事情是我刷B站的时长:
|
||||
|
||||

|
||||

|
||||
|
||||
容易计算得出,我一共看了64天的B站,接近六分之一的时间都在看。虽然我确实有着在干活的时候黑听B站和把B站当作音乐播放器的习惯,但是这个时间未免有点太长了。下一年一定要在这个方面做出一定的改变,将更多的时间放在看书上面去,~~虽然写这句话的时候我就在黑听B站~~。
|
||||
|
||||
|
|
BIN
YaeBlog/source/posts/2023-final/image-20240303165826486.png
(Stored with Git LFS)
BIN
YaeBlog/source/posts/2023-final/image-20240303165826486.webp
Normal file
After Width: | Height: | Size: 16 KiB |
|
@ -70,7 +70,7 @@ tags:
|
|||
|
||||
不过我的B站观看时长再度增长30%,这好吗,这不好,~~有这么多时间刷B站,鬼知道你匆匆在哪了~~。
|
||||
|
||||

|
||||

|
||||
|
||||
### 未来
|
||||
|
||||
|
|
BIN
YaeBlog/source/posts/2024-final/image-20250115171809775.png
(Stored with Git LFS)
BIN
YaeBlog/source/posts/2024-final/image-20250115171809775.webp
Normal file
After Width: | Height: | Size: 38 KiB |
|
@ -44,7 +44,7 @@ date: 2022-07-27 11:34:49
|
|||
|
||||
而且采用 `Git`还有一个好处,采用 `Github`的 `Insight`功能可以轻松的看出大家的贡献值()。
|
||||
|
||||

|
||||

|
||||
|
||||
## 一些技术上的收获
|
||||
|
||||
|
|
BIN
YaeBlog/source/posts/big-homework/1.png
(Stored with Git LFS)
BIN
YaeBlog/source/posts/big-homework/1.webp
Normal file
After Width: | Height: | Size: 16 KiB |
|
@ -131,7 +131,7 @@ Hexo init blog
|
|||
```
|
||||
Hexo会以blog为名称创建一个博客文件夹,这个文件夹的内容为
|
||||
|
||||

|
||||

|
||||
|
||||
`node_modules`文件夹是Hexo需要用到的一些npm依赖包的存放地址,`public`文件夹下是由Hexo渲染产生的静态博客文件,`scaffolds`文件夹是博客用到的模板文件,在默认情况下应该有`draft.md`,`page.md`,`post.md`三个模板文件。`themes`是Hexo中可以使用的主题文件。主题也是Hexo一个非常方便的设计,我们可以方便使用其他人编写的Hexo Themes,让自己的博客在不同的风格之间变换。`source`文件夹就是存放我们写作的博客的地方。一般这里面会有两个子文件夹,`_draft`, `_posts`。我们在里面在创建一个`img`文件夹,把自己的头像图片和网站的图标文件都放在里面,在之后的设置的时候使用。
|
||||
|
||||
|
@ -146,7 +146,7 @@ INFO Hexo is running at http://localhost:4000/ . Press Ctrl+C to stop.
|
|||
|
||||
会在本地运行Hexo自带的一台静态博客服务器。我们用浏览器访问http://localhost:4000, 就可以看见Hexo博客的初始界面
|
||||
|
||||

|
||||

|
||||
|
||||
这便说明安装成功了,~~可以开香槟了~~
|
||||
|
||||
|
|
BIN
YaeBlog/source/posts/build-blog-record/1.png
(Stored with Git LFS)
BIN
YaeBlog/source/posts/build-blog-record/1.webp
Normal file
After Width: | Height: | Size: 30 KiB |
BIN
YaeBlog/source/posts/build-blog-record/2.png
(Stored with Git LFS)
BIN
YaeBlog/source/posts/build-blog-record/2.webp
Normal file
After Width: | Height: | Size: 38 KiB |
|
@ -7,6 +7,7 @@ tags:
|
|||
---
|
||||
|
||||
|
||||
|
||||
我们编译是这样的,在本平台上编译只要敲三条命令就好了,而交叉编译要考虑的就很多了。
|
||||
|
||||
<!--more-->
|
||||
|
@ -45,7 +46,7 @@ tags:
|
|||
|
||||
通常一份GNU工具链只能针对一个平台进行编译,但是LLVM工具链是一套先天的交叉编译工具链,例如对于`llc`工具,使用`llc --version`命令可以看见该编译器可以生成多种目标平台上的汇编代码:
|
||||
|
||||

|
||||

|
||||
|
||||
在使用`clang++`时加上`--target=<triple>`指定目标三元组就可以进行交叉编译。
|
||||
|
||||
|
@ -62,7 +63,7 @@ int main()
|
|||
}
|
||||
```
|
||||
|
||||

|
||||

|
||||
|
||||
看样子交叉编译也不是开箱即用的。最开始我们猜想系统提供的LLVM工具链没有被配置为交叉编译,因此尝试在本地自行编译一套LLVM工具链。
|
||||
|
||||
|
@ -81,7 +82,7 @@ cmake ../llvm-project.src/llvm \
|
|||
|
||||
编译之后的成果会安装到`/usr/local/`目录下,而在`$PATH`环境变量中`/usr/local`位置将在`/usr`目录之前,因此调用时将会优先调用我们自行编译的LLVM工具链,而不是系统中安装的LLVM工具链。
|
||||
|
||||

|
||||

|
||||
|
||||
但是使用这套编译工具链仍然会爆出和之前一样的问题。说明这并不是系统安装LLVM工具链的问题。仔细一想也确实,这里提示找不到对应的头文件应该是找不到RISC-V架构之下的头文件——这里的也是交叉编译的主要问题所在:虽然LLVM工具链宣称自己是原生支持交叉编译的,但是没人宣称说标准库和头文件是原生的。这里我们就需要一个根文件系统来提供这些头文件和各种库文件。
|
||||
|
||||
|
@ -198,7 +199,7 @@ clang++ --target=riscv64-linux-gnu --sysroot=$ROOTFS_DIR -fuse-ld=lld hello.cpp
|
|||
|
||||
第一个问题的回答是Arch Linux安装的LLVM工具是可以交叉编译的。虽然在Arch Linux官方构建LLVM工具链的[构建脚本](https://gitlab.archlinux.org/archlinux/packaging/packages/clang/-/blob/main/PKGBUILD?ref_type=heads)中没有使用`LLVM_TARGETS_TO_BUILD`参数,但是这个参数的默认值是`all`。这一点我们也可以通过实验来验证。
|
||||
|
||||
于是回到编译`llvm`的目录下执行`cat install_manifest.txt | sudo xargs rm`。
|
||||
于是回到编译`llvm`的目录下执行`cat install_manifest.txt | sudo xargs rm`。
|
||||
|
||||
第二个问题的回答可以使用实验来验证,首先安装`riscv64-linux-gnu-gcc`,然后将根文件系统的位置设置为`/usr/riscv64-linux-gnu`,重新编译上面的你好世界样例。编译之后可以正常执行。
|
||||
|
||||
|
@ -229,4 +230,4 @@ export ROOTFS_DIR=<rootfs>
|
|||
|
||||
但是现在的.NET在RISC-V平台上还是废物一个,甚至连`dotnet new`都跑不过,下一步看看能不能运行一下运行时的测试集看看。
|
||||
|
||||

|
||||

|
||||
|
|
BIN
YaeBlog/source/posts/build-dotnet-from-source/image-20240824120646587.png
(Stored with Git LFS)
After Width: | Height: | Size: 78 KiB |
BIN
YaeBlog/source/posts/build-dotnet-from-source/image-20240824121425007.png
(Stored with Git LFS)
After Width: | Height: | Size: 14 KiB |
BIN
YaeBlog/source/posts/build-dotnet-from-source/image-20240824134158262.png
(Stored with Git LFS)
After Width: | Height: | Size: 72 KiB |
BIN
YaeBlog/source/posts/build-dotnet-from-source/image-20240824153514149.png
(Stored with Git LFS)
After Width: | Height: | Size: 14 KiB |
BIN
YaeBlog/source/posts/build-dotnet-from-source/image-20240824214145759.png
(Stored with Git LFS)
After Width: | Height: | Size: 79 KiB |
|
@ -17,7 +17,7 @@ date: 2022-05-08 11:35:19
|
|||
|
||||
我项目的结构大致如图所示:
|
||||
|
||||

|
||||

|
||||
|
||||
在`include`的头文件目录下有两个头文件,`rail.h`和`bus.h`,这两个头文件分别定义了两个结构体`rail_node_t`和`bus_t`。
|
||||
|
||||
|
@ -68,7 +68,7 @@ typedef struct bus bus_t;
|
|||
|
||||
项目的`test`文件夹下是单元测试文件夹,但是在编译的时候会报错
|
||||
|
||||

|
||||

|
||||
|
||||
大意就是在一个google test内部的头文件中有几个函数找不到定义,这个函数都位于`io.h`这个头文件中。
|
||||
|
||||
|
|
BIN
YaeBlog/source/posts/c-include-problems/1.png
(Stored with Git LFS)
BIN
YaeBlog/source/posts/c-include-problems/1.webp
Normal file
After Width: | Height: | Size: 5.4 KiB |
BIN
YaeBlog/source/posts/c-include-problems/2.png
(Stored with Git LFS)
BIN
YaeBlog/source/posts/c-include-problems/2.webp
Normal file
After Width: | Height: | Size: 205 KiB |
|
@ -5,13 +5,14 @@ tags:
|
|||
- 杂谈
|
||||
---
|
||||
|
||||
|
||||
2024年的中国计算机大会于10月24日到10月26日在浙江省金华市东阳市横店镇举办,而鄙人在下不才我,有幸受到实验室资助前去参观学习。
|
||||
|
||||
<!--more-->
|
||||
|
||||
首先开幕式镇楼。
|
||||
|
||||

|
||||

|
||||
|
||||
## 学术上
|
||||
|
||||
|
@ -21,11 +22,11 @@ tags:
|
|||
|
||||
第一个报告是华为庞加莱实验室秦彬娟老师的《异构智算时代的操作系统演进》。报告高屋建瓴,从比较宏观的角度上介绍了当前异构融合操作系统诞生的背景、发展的方向。在报告中重点介绍了一种异构融合操作系统的设计思路:通过三层架构,基于互联池化技术,构建AI时代的融合算力系统。系统中的三层包括:(1)池化基础底层,包括多设备的融合和池化设备虚拟化;(2)异构融合核心子系统,例如异构融合调度系统、异构融合内存和异构融合存储系统;(3)异构核心服务。总的来说,这个报告在一定程度上勾勒出了未来一个异构融合操作系统应有的各项功能,但是显然这一操作系统的实现还存在着明显的困难。
|
||||
|
||||

|
||||

|
||||
|
||||
下面一个报告是较为有干货的报告,北京航空航天大学刘瀚骋老师的《异构融合OS及多样性内存管理框架》。报告中介绍了一个称作`FMMU`的系统,是对于异构融合操作系统中内存管理系统的探索。报告中首先介绍了内存池化技术对于异构融合操作系统的重要性,指出分布式共享内存(Distributed Shared Memory)可能是实现内存池化技术的未来。然后介绍了将部分内存管理中的计算卸载到可编程网络硬件中来加速分布式内存访问的新思路。最后在报告中提到了内存管理技术如何解决错误预测和错误回复的问题。虽然在听的时候没太注意,但是现在总结的时候才发现这个报告的思路似乎有点混乱,尤其是最后一点和内存管理系统并没有什么直接的关系,而且这个内存管理系统似乎不是**异构系统**的内存管理,反而是分布式系统的内存管理。不过总的来说,这个报告还是非常实际的,介绍了不少当前异构融合操作系统中的内存管理面临的问题和解决问题的探索。
|
||||
|
||||

|
||||

|
||||
|
||||
第三个报告是国防科技大学李东升老师的《异构计算环境下的分布式深度学习训练》。报告首先从李老师的主业——并行计算起手,介绍了深度学习训练过程中主要的各种并行方法,例如数据并行、模型并行和混合并行等,指出目前大模型的并行训练存在着计算/存储/通信难的问题。因此,提出了一个智能模型训练并行任务划分方法:(1)基于符号算子的计算图定义方法;(2)面向Transformer模型的流水线并行任务划分方法;(3)异构资源感知的流水线并行任务划分方法。然后针对分布式模型训练中通信调度存在的通信墙、数据依赖关系复杂等的问题,提出综合词嵌入表的稀疏通信调度技术、流水线并行的P2P通信调度技术、模型计算的统一操作执行引擎和网络链路感知的通信执行引擎的通信调度技术。最后提到了智能模型训练 的内存优化技术,针对现有重计算技术(re-computing)和存储交换(swapping)技术存在的问题,提出了一种面向大型智能模型训练的细粒度内存优化方法`DELTA`。
|
||||
|
||||
|
@ -49,7 +50,7 @@ Plane讨论没有参加。
|
|||
|
||||
第二个报告是南京大学冯新宇老师的《基于仓颉语言的嵌入式DSL开发》,同时冯新宇老师也是仓颉语言的首席架构师。冯老师的这个报告主要聚焦于仓颉语言提供的嵌入式DSL能力,而嵌入式DSL这一设计范式已经在前端开发中展现了不俗的潜力。报告中介绍了嵌入式DSL出现的背景,仓颉中为了提供嵌入式DSL而引入的语法糖、仓颉提供的嵌入式DSL工具箱等。虽然仓颉语言是一个主要面向上层应用开发的语言,但是仓颉中丰富的DSL能力还是给异构编程模型的设计提供了不少的启发。而且目前在各种深度学习编译器中DSL的应用也非常广泛,例如`triton`。
|
||||
|
||||

|
||||

|
||||
|
||||
第三个报告是在存算一体的芯片上做数据库的加速,第四个报告是OpenHarmony上`ArkTS`程序的静态分析,都没怎么听。
|
||||
|
||||
|
|
BIN
YaeBlog/source/posts/cncc-2024/image-20241102211959206.png
(Stored with Git LFS)
BIN
YaeBlog/source/posts/cncc-2024/image-20241102211959206.webp
Normal file
After Width: | Height: | Size: 136 KiB |
BIN
YaeBlog/source/posts/cncc-2024/image-20241102212355390.png
(Stored with Git LFS)
BIN
YaeBlog/source/posts/cncc-2024/image-20241102212355390.webp
Normal file
After Width: | Height: | Size: 238 KiB |
BIN
YaeBlog/source/posts/cncc-2024/image-20241102212536635.png
(Stored with Git LFS)
BIN
YaeBlog/source/posts/cncc-2024/image-20241102212536635.webp
Normal file
After Width: | Height: | Size: 47 KiB |
BIN
YaeBlog/source/posts/cncc-2024/image-20241102212738598.png
(Stored with Git LFS)
BIN
YaeBlog/source/posts/cncc-2024/image-20241102212738598.webp
Normal file
After Width: | Height: | Size: 639 KiB |
|
@ -198,7 +198,7 @@ bazel build -c opt --strip=ALWAYS \
|
|||
|
||||
如果在编译的过程中提示缺失`dx.jar`这个文件而且你用的SDK版本还是高于31的,那可能是SDK中缺失了这个文件,可以将SDk降级到30就含有这个文件了。我使用的解决办法比较离奇,我是将30版本的SDK文件中的这个文件软链接过来,解决了这个问题。
|
||||
|
||||

|
||||

|
||||
|
||||
编译消耗的时间可能比较的长,耐心等待即可。
|
||||
|
||||
|
@ -227,7 +227,7 @@ bazel build -c opt //mediapipe/graphs/pose_tracking:pose_tracking_gpu_binary_gra
|
|||
|
||||
然后还需要从服务器上下载`tflite`文件,`Pose Tracking`这个解决方案需要两个`tflite`文件,第一个是[pose_detection.tflite](https://storage.googleapis.com/mediapipe-assets/pose_detection.tflite),第二个文件则有三个不同的选择,分别对于解决方案中提供的三个质量版本:
|
||||
|
||||

|
||||

|
||||
|
||||
下载地址是[pose_landmark_full.tflite](https://storage.googleapis.com/mediapipe-assets/pose_landmark_full.tflite),[pose_landmark_heavy.tflite](https://storage.googleapis.com/mediapipe-assets/pose_landmark_heavy.tflite)和[pose_landmark_lite.tflite](https://storage.googleapis.com/mediapipe-assets/pose_landmark_lite.tflite)。
|
||||
|
||||
|
|
BIN
YaeBlog/source/posts/compile-mediapipe/2023-01-15-22-05-41-Screenshot_20230115_220521.png
(Stored with Git LFS)
After Width: | Height: | Size: 36 KiB |
BIN
YaeBlog/source/posts/compile-mediapipe/2023-01-19-20-20-40-Screenshot_20230119_202008.png
(Stored with Git LFS)
After Width: | Height: | Size: 7.9 KiB |
|
@ -1,20 +1,21 @@
|
|||
---
|
||||
title: 计算机系统结构——流水线复习
|
||||
date: 2024-06-12T20:27:25.0000000
|
||||
tags:
|
||||
- 计算机系统结构
|
||||
- 学习资料
|
||||
date: 2024-06-12 20:27:25
|
||||
- 计算机系统结构
|
||||
- 学习资料
|
||||
---
|
||||
|
||||
|
||||
让指令的各个执行阶段依次进行运行是一个简单而自然的想法,但是这种方式执行速度慢、运行效率低。因此一个很自然的想法就是将指令重叠起来运行,让执行功能部件被充分的利用起来,这就是**流水线**。
|
||||
|
||||
流水线的表示方法有两种。
|
||||
|
||||

|
||||

|
||||
|
||||
第一种被称作**连接图**,清晰的表达出了流水线内部的逻辑关系。
|
||||
|
||||

|
||||

|
||||
|
||||
> 上图中给出了两个流水线中的概念:通过时间和排空时间。其中通过时间又被称作装入时间,是指第一个任务进入流水线到完成的事件;排空时间则相反,是最后一个任务通过流水线的时间。
|
||||
|
||||
|
@ -40,7 +41,7 @@ date: 2024-06-12 20:27:25
|
|||
- 静态流水线,同一时间内,多功能流水线的各段只能按照同一种功能的方式连接。
|
||||
- 动态流水线,同一时间内,多功能流水线的各种可以按照不同的方式连接,执行不同的功能。
|
||||
|
||||

|
||||

|
||||
|
||||
按照流水线中是否存在反馈回路分类:
|
||||
|
||||
|
@ -58,7 +59,7 @@ date: 2024-06-12 20:27:25
|
|||
- 加速比,同一任务,不使用流水线所使用时间与使用流水线所用时间比。
|
||||
- 效率,流水线设备的利用率。
|
||||
|
||||

|
||||

|
||||
|
||||
在设计流水线的过程中存在若干问题。
|
||||
|
||||
|
@ -68,7 +69,7 @@ date: 2024-06-12 20:27:25
|
|||
|
||||
一个典型的五段流水线MIPS流水线:
|
||||
|
||||

|
||||

|
||||
|
||||
|
||||
|
||||
|
|
BIN
YaeBlog/source/posts/computer-architecture-pipeline/image-20240612184855300.png
(Stored with Git LFS)
After Width: | Height: | Size: 8.4 KiB |
BIN
YaeBlog/source/posts/computer-architecture-pipeline/image-20240612184949777.png
(Stored with Git LFS)
After Width: | Height: | Size: 24 KiB |
BIN
YaeBlog/source/posts/computer-architecture-pipeline/image-20240612190426368.png
(Stored with Git LFS)
After Width: | Height: | Size: 23 KiB |
BIN
YaeBlog/source/posts/computer-architecture-pipeline/image-20240612192700169.png
(Stored with Git LFS)
After Width: | Height: | Size: 14 KiB |
BIN
YaeBlog/source/posts/computer-architecture-pipeline/image-20240612193301372.png
(Stored with Git LFS)
After Width: | Height: | Size: 57 KiB |
|
@ -92,7 +92,7 @@ sudo systemctl enable sddm.service
|
|||
|
||||
我目前实现的效果大概长这样:
|
||||
|
||||

|
||||

|
||||
|
||||
颇有一种`Windows`和`MacOS`杂交的风格。
|
||||
|
||||
|
@ -106,7 +106,7 @@ sudo systemctl enable sddm.service
|
|||
|
||||
先上一张`shell`的系统概览截图:
|
||||
|
||||

|
||||

|
||||
|
||||
终端模拟器直接使用的`konsole`,目前没有进行改动。
|
||||
|
||||
|
|
BIN
YaeBlog/source/posts/daily-linux-0/2023-01-12-13-28-38-Screenshot_20230112_132829.png
(Stored with Git LFS)
After Width: | Height: | Size: 138 KiB |
BIN
YaeBlog/source/posts/daily-linux-0/2023-01-12-13-36-45-Screenshot_20230112_133628.png
(Stored with Git LFS)
After Width: | Height: | Size: 73 KiB |
|
@ -18,7 +18,7 @@ typora-root-url: daily-linux-2
|
|||
|
||||
最近恰好被平铺式的窗口管理器种草,又在B站上看见一个动画绚丽的`wayland`合成器——[Hyprland](https://hyprland.org/),当即脑袋一热,就把`kde`干掉,装上了`hyprland`。
|
||||
|
||||

|
||||

|
||||
|
||||
安装`hyprland`的过程非常舒适,`hyprland`被打包为一个单独的二进制文件,使用`pacman`安装之后直接在`tty`下执行:
|
||||
|
||||
|
@ -46,7 +46,7 @@ Hyprland
|
|||
|
||||
各种在学习过程中遇到的工具软件:基本上都工作运行良好。当然因为没有设置缩放的问题而导致字体都很小。因为如果在配置文件中设置缩放之后会导致字体发虚。下面的截图就是我将我的2K显示屏设置为150%缩放的效果,~~虽然在截图中的效果不明显~~。目前在常用软件中唯一让我十分不满意的软件是`wps`,使用体验完全无法和`offices`相提并论,目前我正在研究使用`wine`运行`offices`,如果成功了就再水一篇博客庆祝一下。
|
||||
|
||||

|
||||

|
||||
|
||||
> 最新的进展是使用`wine`没法安装学校提供的`office 2021`,同时我又不愿意使用古老的`office`版本,但是我发现一个称作`onlyoffice`的第三方软件蛮好用的,等我试用一段时间再说。
|
||||
>
|
||||
|
|
BIN
YaeBlog/source/posts/daily-linux-2/df4211f6be2724b3b4725f7ce5a4078818844857.jpg
(Stored with Git LFS)
BIN
YaeBlog/source/posts/daily-linux-2/image-20230702205919301.png
(Stored with Git LFS)
BIN
YaeBlog/source/posts/daily-linux-2/image-20230702205919301.webp
Normal file
After Width: | Height: | Size: 53 KiB |
|
@ -53,7 +53,7 @@ date: 2023-09-04 14:47:46
|
|||
- `Meta+F`全屏应用
|
||||
- `Meta+W`关闭应用
|
||||
|
||||

|
||||

|
||||
|
||||
### Fuck You NVIDIA
|
||||
|
||||
|
|
BIN
YaeBlog/source/posts/daily-linux-3/Screenshot_20230904_144149.png
(Stored with Git LFS)
After Width: | Height: | Size: 155 KiB |
|
@ -1,11 +1,12 @@
|
|||
---
|
||||
title: 日用Linux挑战 第4篇 新的开始
|
||||
date: 2024-03-09T14:00:00.0000000
|
||||
tags:
|
||||
- Linux
|
||||
- 杂谈
|
||||
date: 2024/03/09 14:00:00
|
||||
- Linux
|
||||
- 杂谈
|
||||
---
|
||||
|
||||
|
||||
小步快跑,面向未来。
|
||||
|
||||
<!--more-->
|
||||
|
@ -18,7 +19,7 @@ date: 2024/03/09 14:00:00
|
|||
|
||||
相较于古老但是稳定的`Ext4`文件系统,`Btrfs`对我来说最大的好处便是可以零成本的创建快照,便于在出现错误的时候及时回滚或者直接重装系统。因此,为了方便快照的生成和回滚,我在安装系统时使用**扁平化**的子分区划分方法:即尽力避免出现嵌套的子分区,所有需要快照的分区都处在`/`目录之下:
|
||||
|
||||

|
||||

|
||||
|
||||
- `@`为根分区,挂载在`/`目录之下,打开写时复制;
|
||||
- `@home`为家目录分区,挂载在`/home`目录之下,打开写时复制;
|
||||
|
@ -44,7 +45,7 @@ sudo btrfs send /.snapshots/home@20240225 | zstd | ssh root@remote "zstd -d | bt
|
|||
|
||||
算起来,我已经和`Wayland`显示协议相爱相杀了整整一年了,从`KDE plasma X`到`Hyprland`,再尝试小众的`labwc`,最后回到了`KDE plasma X`。而在2024年2月29日`KDE plasma`释出6.0版本,将`Wayland`作为默认的显示协议,我也在第一时间更新了版本并使用`wayland`显示协议。现在,我可以比较确定的说,`Wayland`目前已经达到可用的水平了,而且我还是使用`RTX 3060`显卡。
|
||||
|
||||

|
||||

|
||||
|
||||
不过相较于`AMDGPU`可以开箱即用,使用`NVIDIA`启动需要配置如下的模块参数:
|
||||
|
||||
|
@ -64,7 +65,7 @@ options nvidia_drm modeset=1 fbdev=1
|
|||
|
||||
不过`XWayland`应用程序在使用`NVIDIA`驱动时会存在一个神奇的**同步失败**问题,表现为在`xwayland`中部分控件闪烁,交替显示更新前和更新后的帧,而且这个问题几乎不能被截屏抓到,具体可以见`freedesktop`上的这个[issue](https://gitlab.freedesktop.org/xorg/xserver/-/issues/1317)。虽然这个议题下面有着很长的讨论,还是建议大家完整的看一遍,里面甚至还有:
|
||||
|
||||

|
||||

|
||||
|
||||
省流:这个议题讨论了在`xserver`中提供显式同步的协议原语,方便图形驱动程序知道什么时候渲染的帧发生了变化。因此这并不是一个`NVIDIA`驱动程序的问题,而是需要将`Linux`显示协议栈从隐式同步迁移到显式同步。但是相关的工作还在开发过程中,因此解决方法有两个:
|
||||
|
||||
|
@ -77,9 +78,9 @@ options nvidia_drm modeset=1 fbdev=1
|
|||
|
||||
于是,我就在`Arch Wiki`上学到一条新知识:
|
||||
|
||||

|
||||

|
||||
|
||||
原来`efi`分区其实只用放`grub`,,,
|
||||
|
||||

|
||||

|
||||
|
||||
|
|
BIN
YaeBlog/source/posts/daily-linux-4/Screenshot_20240309_115143.png
(Stored with Git LFS)
After Width: | Height: | Size: 39 KiB |
BIN
YaeBlog/source/posts/daily-linux-4/cfd17cff0701a8e8c69fecf247f17fc1-1709963611271-2.jpg
(Stored with Git LFS)
After Width: | Height: | Size: 197 KiB |
BIN
YaeBlog/source/posts/daily-linux-4/image-20240309130329784.png
(Stored with Git LFS)
BIN
YaeBlog/source/posts/daily-linux-4/image-20240309130329784.webp
Normal file
After Width: | Height: | Size: 104 KiB |
BIN
YaeBlog/source/posts/daily-linux-4/image-20240309131750535.png
(Stored with Git LFS)
BIN
YaeBlog/source/posts/daily-linux-4/image-20240309131750535.webp
Normal file
After Width: | Height: | Size: 60 KiB |