Mirrors/roadie
2
0
Fork 0
mirror of https://github.com/sphildreth/roadie synced 2024-11-10 06:44:12 +00:00
Code Issues Projects Releases Packages Wiki Activity

CodeMaid cleanup, Warning resolve work

Browse source
This commit is contained in:
Steven Hildreth 2022-01-18 16:52:02 -06:00
parent 022920b3f3
commit 0fd9fc92d0
45 changed files with 652 additions and 190 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Roadie.Api.Library.Tests/ArtistLookupEngineTests.cs
View file

@ -20,7 +20,9 @@ namespace Roadie.Library.Tests
}
private IRoadieSettings Configuration { get; }
public DictionaryCacheManager CacheManager { get; }
private Encoding.IHttpEncoder HttpEncoder { get; }
public ArtistLookupEngineTests()

111
Roadie.Api.Library.Tests/LastFmHelperTests.cs Normal file
View file

@ -0,0 +1,111 @@
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using Roadie.Library.Caching;
using Roadie.Library.Configuration;
using Roadie.Library.MetaData.LastFm;
using Roadie.Library.Processors;
using Roadie.Library.MetaData.MusicBrainz;
using Roadie.Library.SearchEngines.MetaData.Discogs;
using System;
using System.Diagnostics;
using System.Linq;
using System.Threading.Tasks;
using Xunit;
using Roadie.Library.Data.Context;
using System.Collections.Generic;
namespace Roadie.Library.Tests
{
public class LastFmHelperTests : HttpClientFactoryBaseTests
{
private IEventMessageLogger MessageLogger { get; }
private IRoadieSettings Configuration { get; }
public DictionaryCacheManager CacheManager { get; }
private Encoding.IHttpEncoder HttpEncoder { get; }
private IRoadieDbContext RoadieDbContext { get; }
private ILogger Logger
{
get
{
return MessageLogger as ILogger;
}
}
public LastFmHelperTests()
{
MessageLogger = new EventMessageLogger<SearchEngineTests>();
MessageLogger.Messages += MessageLogger_Messages;
var settings = new RoadieSettings();
IConfigurationBuilder configurationBuilder = new ConfigurationBuilder();
configurationBuilder.AddJsonFile("appsettings.test.json");
IConfiguration configuration = configurationBuilder.Build();
configuration.GetSection("RoadieSettings").Bind(settings);
Configuration = settings;
CacheManager = new DictionaryCacheManager(Logger, new SystemTextCacheSerializer(Logger), new CachePolicy(TimeSpan.FromHours(4)));
HttpEncoder = new Encoding.DummyHttpEncoder();
}
[Fact]
public async Task LastFMReleaseSearch()
{
if (!Configuration.Integrations.LastFmProviderEnabled)
{
return;
}
var logger = new EventMessageLogger<LastFmHelper>();
logger.Messages += MessageLogger_Messages;
var lfmHelper = new LastFmHelper(Configuration, CacheManager, new EventMessageLogger<LastFmHelper>(), RoadieDbContext, HttpEncoder, _httpClientFactory);
var artistName = "Billy Joel";
var title = "Piano Man";
var sw = Stopwatch.StartNew();
var result = await lfmHelper.PerformReleaseSearch(artistName, title, 1).ConfigureAwait(false);
sw.Stop();
Assert.NotNull(result);
Assert.NotNull(result.Data);
Assert.NotEmpty(result.Data);
var release = result.Data.FirstOrDefault();
Assert.NotNull(release);
}
[Fact]
public async Task LastFMArtistSearch()
{
if (!Configuration.Integrations.LastFmProviderEnabled)
{
return;
}
var logger = new EventMessageLogger<LastFmHelper>();
logger.Messages += MessageLogger_Messages;
var lfmHelper = new LastFmHelper(Configuration, CacheManager, new EventMessageLogger<LastFmHelper>(), RoadieDbContext, HttpEncoder, _httpClientFactory);
var artistName = "Billy Joel";
var sw = Stopwatch.StartNew();
var result = await lfmHelper.PerformArtistSearchAsync(artistName, 1).ConfigureAwait(false);
sw.Stop();
Assert.NotNull(result);
Assert.NotNull(result.Data);
Assert.NotEmpty(result.Data);
var release = result.Data.FirstOrDefault();
Assert.NotNull(release);
}
private void MessageLogger_Messages(object sender, EventMessage e)
{
Console.WriteLine($"Log Level [{ e.Level }] Log Message [{ e.Message }] ");
}
}
}

2
Roadie.Api.Library.Tests/Roadie.Library.Tests.csproj
View file

@ -19,7 +19,7 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.AspNet.WebApi.Core" Version="5.2.7" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.WebApiCompatShim" Version="2.2.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="6.0.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.0.0" />
<PackageReference Include="Moq" Version="4.16.1" />

51
Roadie.Api.Library/Caching/SystemTextCacheSerializer.cs Normal file
View file

@ -0,0 +1,51 @@
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Generic;
using System.Text;
namespace Roadie.Library.Caching
{
public sealed class SystemTextCacheSerializer : ICacheSerializer
{
private ILogger Logger { get; }
public SystemTextCacheSerializer(ILogger logger)
{
Logger = logger;
}
public string Serialize(object o)
{
if (o == null)
{
return null;
}
try
{
return System.Text.Json.JsonSerializer.Serialize(o);
}
catch (Exception ex)
{
Logger.LogError(ex);
}
return null;
}
public TOut Deserialize<TOut>(string s)
{
if (string.IsNullOrEmpty(s))
{
return default(TOut);
}
try
{
return System.Text.Json.JsonSerializer.Deserialize<TOut>(s);
}
catch (Exception ex)
{
Logger.LogError(ex);
}
return default(TOut);
}
}
}

2
Roadie.Api.Library/Extensions/ExceptionExt.cs
View file

@ -15,7 +15,7 @@ namespace Roadie.Library.Extensions
{
return JsonSerializer.Serialize(input, new JsonSerializerOptions
{
IgnoreNullValues = true,
DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull,
WriteIndented = true
});
}

1
Roadie.Api.Library/Roadie.Library.csproj
View file

@ -15,6 +15,7 @@
<PackageReference Include="EFCore.BulkExtensions" Version="6.3.0" />
<PackageReference Include="FluentFTP" Version="36.1.0" />
<PackageReference Include="Hashids.net" Version="1.4.1" />
<PackageReference Include="Hqub.Last.fm" Version="2.1.0" />
<PackageReference Include="HtmlAgilityPack" Version="1.11.40" />
<PackageReference Include="IdSharp.Common" Version="1.0.1" />
<PackageReference Include="IdSharp.Tagging" Version="1.0.0-rc3" />

2
Roadie.Api.Library/Roadie.Library.nuspec
View file

@ -2,7 +2,7 @@
<package >
<metadata>
<id>$id$</id>
<version>$version$</version>
<version>1.0.1-pre</version>
<title>$title$</title>
<authors>$author$</authors>
<requireLicenseAcceptance>false</requireLicenseAcceptance>

95
Roadie.Api.Library/SearchEngines/MetaData/LastFm/JsonEntities.cs Normal file
View file

@ -0,0 +1,95 @@
using System.Text.Json.Serialization;
namespace Roadie.Library.SearchEngines.MetaData.LastFm
{
public class Rootobject
{
public Album album { get; set; }
}
public class Album
{
public string artist { get; set; }
public string mbid { get; set; }
public Tags tags { get; set; }
public string name { get; set; }
public Image[] image { get; set; }
public Tracks tracks { get; set; }
public string listeners { get; set; }
public string playcount { get; set; }
public string url { get; set; }
}
public class Tags
{
public Tag[] tag { get; set; }
}
public class Tag
{
public string url { get; set; }
public string name { get; set; }
}
public class Tracks
{
public Track[] track { get; set; }
}
public class Track
{
public Streamable streamable { get; set; }
public int duration { get; set; }
public string url { get; set; }
public string name { get; set; }
[JsonPropertyName("@attr")]
public Attr attr { get; set; }
//public int? TrackNumber => string.IsNullOrWhiteSpace(attr) ? null : int.Parse(attr.Replace("\"@attr\":{\"rank\":", "").Replace("}", ""));
public int? TrackNumber => attr?.rank;
public Artist artist { get; set; }
}
public class Streamable
{
public string fulltrack { get; set; }
public string text { get; set; }
}
public class Attr
{
public int rank { get; set; }
}
public class Artist
{
public string url { get; set; }
public string name { get; set; }
public string mbid { get; set; }
}
public class Image
{
public string size { get; set; }
public string text { get; set; }
}
}

142
Roadie.Api.Library/SearchEngines/MetaData/LastFm/LastFmHelper.cs
View file

@ -1,7 +1,6 @@
using IF.Lastfm.Core.Api;
using IF.Lastfm.Core.Objects;
using Microsoft.Extensions.Logging;
using RestSharp;
using Roadie.Library.Caching;
using Roadie.Library.Configuration;
using Roadie.Library.Data.Context;
@ -18,20 +17,22 @@ using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Security.Cryptography;
using System.Text;
using System.Threading.Tasks;
using System.Xml;
using System.Xml.XPath;
using data = Roadie.Library.Data;
using static System.Net.Mime.MediaTypeNames;
namespace Roadie.Library.MetaData.LastFm
{
public class LastFmHelper : MetaDataProviderBase, ILastFmHelper
{
private const string LastFmErrorCodeXPath = "/lfm/error/@code";
private const string LastFmErrorXPath = "/lfm/error";
private const string LastFmStatusOk = "ok";
private const string LastFmStatusXPath = "/lfm/@status";
public override bool IsEnabled =>
@ -90,12 +91,17 @@ namespace Roadie.Library.MetaData.LastFm
{
{"token", token}
};
var request = new RestRequest();
request.Method = Method.Get;
var client = new RestClient(BuildUrl("auth.getSession", parameters));
var responseXML = await client.ExecuteAsync<string>(request).ConfigureAwait(false);
string responseXML = null;
var client = _httpClientFactory.CreateClient();
var request = new HttpRequestMessage(HttpMethod.Get, BuildUrl("auth.getSession", parameters));
request.Headers.Add("User-Agent", WebHelper.UserAgent);
var response = await client.SendAsync(request).ConfigureAwait(false);
if (response.IsSuccessStatusCode)
{
responseXML = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
}
var doc = new XmlDocument();
doc.LoadXml(responseXML.Content);
doc.LoadXml(responseXML);
var sessionKey = doc.GetElementsByTagName("key")[0].InnerText;
return new OperationResult<string>
{
@ -117,8 +123,6 @@ namespace Roadie.Library.MetaData.LastFm
{
return new OperationResult<bool>("User does not have LastFM Integration setup");
}
await Task.Run(() =>
{
var method = "track.updateNowPlaying";
var parameters = new RequestParameters
{
@ -132,21 +136,18 @@ namespace Roadie.Library.MetaData.LastFm
parameters.Add("api_sig", signature);
ServicePointManager.Expect100Continue = false;
var request = WebRequest.Create(url);
request.Method = "POST";
var postData = parameters.ToString();
var byteArray = System.Text.Encoding.UTF8.GetBytes(postData);
request.ContentType = "application/x-www-form-urlencoded";
request.ContentLength = byteArray.Length;
using (var dataStream = request.GetRequestStream())
var client = _httpClientFactory.CreateClient();
XPathNavigator xp = null;
var parametersJson = new StringContent(CacheManager.CacheSerializer.Serialize(parameters), System.Text.Encoding.UTF8, Application.Json);
using (var httpResponseMessage = await client.PostAsync(url, parametersJson).ConfigureAwait(false))
{
dataStream.Write(byteArray, 0, byteArray.Length);
dataStream.Close();
}
var xp = GetResponseAsXml(request);
Logger.LogTrace($"LastFmHelper: RoadieUser `{roadieUser}` NowPlaying `{scrobble}` LastFmResult [{xp.InnerXml}]");
if (httpResponseMessage.IsSuccessStatusCode)
{
xp = await GetResponseAsXml(httpResponseMessage).ConfigureAwait(false);
result = true;
}).ConfigureAwait(false);
}
}
Logger.LogTrace($"LastFmHelper: Success [{ result }] RoadieUser `{roadieUser}` NowPlaying `{scrobble}` LastFmResult [{xp.InnerXml}]");
}
catch (Exception ex)
{
@ -169,7 +170,7 @@ namespace Roadie.Library.MetaData.LastFm
{
Logger.LogTrace("LastFmHelper:PerformArtistSearch:{0}", query);
var auth = new LastAuth(ApiKey.Key, ApiKey.KeySecret);
var albumApi = new ArtistApi(auth);
var albumApi = new ArtistApi(auth, _httpClientFactory.CreateClient());
var response = await albumApi.GetInfoAsync(query).ConfigureAwait(false);
if (!response.Success)
{
@ -210,14 +211,24 @@ namespace Roadie.Library.MetaData.LastFm
var cacheKey = $"uri:lastfm:releasesearch:{ artistName.ToAlphanumericName() }:{ query.ToAlphanumericName() }";
var data = await CacheManager.GetAsync<ReleaseSearchResult>(cacheKey, async () =>
{
var request = new RestRequest();
request.Method = Method.Get;
var client = new RestClient(string.Format("http://ws.audioscrobbler.com/2.0/?method=album.getinfo&api_key={0}&artist={1}&album={2}&format=xml", ApiKey.Key, artistName, query));
var responseData = await client.ExecuteAsync<lfm>(request).ConfigureAwait(false);
Rootobject response = null;
var client = _httpClientFactory.CreateClient();
var request = new HttpRequestMessage(HttpMethod.Get, $"http://ws.audioscrobbler.com/2.0/?method=album.getinfo&api_key={ ApiKey.Key }&artist={ artistName }&album={ query }&format=json");
request.Headers.Add("User-Agent", WebHelper.UserAgent);
var sendResponse = await client.SendAsync(request).ConfigureAwait(false);
if (sendResponse.IsSuccessStatusCode)
{
try
{
var r = await sendResponse.Content.ReadAsStringAsync().ConfigureAwait(false);
response = CacheManager.CacheSerializer.Deserialize<Rootobject>(r);
}
catch (Exception ex)
{
Logger.LogError(ex);
}
}
ReleaseSearchResult result = null;
var response = responseData != null && responseData.Data != null ? responseData.Data : null;
if (response != null && response.album != null)
{
var lastFmAlbum = response.album;
@ -229,15 +240,15 @@ namespace Roadie.Library.MetaData.LastFm
// No longer fetching/consuming images LastFm says is violation of ToS ; https://getsatisfaction.com/lastfm/topics/api-announcement-dac8oefw5vrxq
if (lastFmAlbum.tags != null) result.Tags = lastFmAlbum.tags.Select(x => x.name).ToList();
if (lastFmAlbum.tags != null) result.Tags = lastFmAlbum.tags.tag.Select(x => x.name).ToList();
if (lastFmAlbum.tracks != null)
{
var tracks = new List<TrackSearchResult>();
foreach (var lastFmTrack in lastFmAlbum.tracks)
foreach (var lastFmTrack in lastFmAlbum.tracks.track)
{
tracks.Add(new TrackSearchResult
{
TrackNumber = SafeParser.ToNumber<short?>(lastFmTrack.rank),
TrackNumber = SafeParser.ToNumber<short?>(lastFmTrack.TrackNumber),
Title = lastFmTrack.name,
Duration = SafeParser.ToNumber<int?>(lastFmTrack.duration),
Urls = string.IsNullOrEmpty(lastFmTrack.url) ? new[] { lastFmTrack.url } : null
@ -282,7 +293,9 @@ namespace Roadie.Library.MetaData.LastFm
var user = DbContext.Users.FirstOrDefault(x => x.RoadieId == roadieUser.UserId);
if (user == null || string.IsNullOrEmpty(user.LastFMSessionKey))
{
return new OperationResult<bool>("User does not have LastFM Integration setup");
}
var parameters = new RequestParameters
{
{"artist", scrobble.ArtistName},
@ -299,19 +312,17 @@ namespace Roadie.Library.MetaData.LastFm
parameters.Add("api_sig", signature);
ServicePointManager.Expect100Continue = false;
var request = WebRequest.Create(url);
request.Method = "POST";
var postData = parameters.ToString();
var byteArray = System.Text.Encoding.UTF8.GetBytes(postData);
request.ContentType = "application/x-www-form-urlencoded";
request.ContentLength = byteArray.Length;
using (var dataStream = request.GetRequestStream())
var client = _httpClientFactory.CreateClient();
XPathNavigator xp = null;
var parametersJson = new StringContent(CacheManager.CacheSerializer.Serialize(parameters), System.Text.Encoding.UTF8, Application.Json);
using (var httpResponseMessage = await client.PostAsync(url, parametersJson).ConfigureAwait(false))
{
dataStream.Write(byteArray, 0, byteArray.Length);
dataStream.Close();
if (httpResponseMessage.IsSuccessStatusCode)
{
xp = await GetResponseAsXml(httpResponseMessage).ConfigureAwait(false);
result = true;
}
}
var xp = GetResponseAsXml(request);
Logger.LogTrace($"LastFmHelper: RoadieUser `{roadieUser}` Scrobble `{scrobble}` LastFmResult [{xp.InnerXml}]");
result = true;
}
@ -389,7 +400,7 @@ namespace Roadie.Library.MetaData.LastFm
return result;
}
protected internal virtual XPathNavigator GetResponseAsXml(WebRequest request)
protected internal virtual XPathNavigator GetResponseXml(HttpWebRequest request)
{
WebResponse response;
XPathNavigator navigator;
@ -413,6 +424,45 @@ namespace Roadie.Library.MetaData.LastFm
return navigator;
}
protected internal virtual async Task<XPathNavigator> GetResponseAsXml(HttpResponseMessage request)
{
XPathNavigator navigator;
try
{
navigator = await GetXpathDocumentFromResponse(request);
CheckLastFmStatus(navigator);
}
catch (WebException exception)
{
var response = exception.Response;
XPathNavigator document;
TryGetXpathDocumentFromResponse(response, out document);
if (document != null) CheckLastFmStatus(document, exception);
throw; // throw even if Last.fm status is OK
}
return navigator;
}
protected virtual async Task<XPathNavigator> GetXpathDocumentFromResponse(HttpResponseMessage response)
{
using (var stream = await response.Content.ReadAsStreamAsync())
{
if (stream == null) throw new InvalidOperationException("Response Stream is null");
try
{
return new XPathDocument(stream).CreateNavigator();
}
catch (XmlException exception)
{
throw new XmlException("Could not read HTTP Response as XML", exception);
}
}
}
protected virtual XPathNavigator GetXpathDocumentFromResponse(WebResponse response)
{
using (var stream = response.GetResponseStream())

6
Roadie.Api.Library/SearchEngines/MetaData/MusicBrainz/MusicBrainzProvider.cs
View file

@ -25,7 +25,7 @@ namespace Roadie.Library.MetaData.MusicBrainz
IHttpClientFactory httpClientFactory)
: base(configuration, cacheManager, logger, httpClientFactory)
{
Repository = new MusicBrainzRepository(configuration, logger);
Repository = new MusicBrainzRepository(configuration, logger, httpClientFactory);
}
public async Task<IEnumerable<AudioMetaData>> MusicBrainzReleaseTracksAsync(string artistName, string releaseTitle)
@ -61,7 +61,7 @@ namespace Roadie.Library.MetaData.MusicBrainz
}
// Now get The Release Details
release = await MusicBrainzRequestHelper.GetAsync<Release>(MusicBrainzRequestHelper.CreateLookupUrl("release", ReleaseResult.MusicBrainzId, "recordings")).ConfigureAwait(false);
release = await MusicBrainzRequestHelper.GetAsync<Release>(_httpClientFactory, MusicBrainzRequestHelper.CreateLookupUrl("release", ReleaseResult.MusicBrainzId, "recordings")).ConfigureAwait(false);
if (release == null) return null;
CacheManager.Add(ReleaseCacheKey, release);
}
@ -300,7 +300,7 @@ namespace Roadie.Library.MetaData.MusicBrainz
};
}
private Task<CoverArtArchivesResult> CoverArtForMusicBrainzReleaseByIdAsync(string musicBrainzId) => MusicBrainzRequestHelper.GetAsync<CoverArtArchivesResult>(MusicBrainzRequestHelper.CreateCoverArtReleaseUrl(musicBrainzId));
private Task<CoverArtArchivesResult> CoverArtForMusicBrainzReleaseByIdAsync(string musicBrainzId) => MusicBrainzRequestHelper.GetAsync<CoverArtArchivesResult>(_httpClientFactory, MusicBrainzRequestHelper.CreateCoverArtReleaseUrl(musicBrainzId));
private async Task<IEnumerable<Release>> ReleasesForArtistAsync(string artist, string artistMusicBrainzId = null)
{

17
Roadie.Api.Library/SearchEngines/MetaData/MusicBrainz/MusicBrainzRepository.cs
View file

@ -13,9 +13,15 @@ namespace Roadie.Library.MetaData.MusicBrainz
public class MusicBrainzRepository
{
private string FileName { get; }
private ILogger<MusicBrainzProvider> Logger { get; }
public MusicBrainzRepository(IRoadieSettings configuration, ILogger<MusicBrainzProvider> logger)
private IHttpClientFactory HttpClientFactory { get; }
public MusicBrainzRepository(
IRoadieSettings configuration,
ILogger<MusicBrainzProvider> logger,
IHttpClientFactory httpClientFactory)
{
Logger = logger;
var location = System.Reflection.Assembly.GetEntryAssembly().Location;
@ -25,6 +31,7 @@ namespace Roadie.Library.MetaData.MusicBrainz
Directory.CreateDirectory(directory);
}
FileName = Path.Combine(directory, "MusicBrainzRespository.db");
HttpClientFactory = httpClientFactory;
}
/// <summary>
@ -47,14 +54,14 @@ namespace Roadie.Library.MetaData.MusicBrainz
if (artist == null)
{
// Perform a query to get the MbId for the Name
var artistResult = await MusicBrainzRequestHelper.GetAsync<ArtistResult>(MusicBrainzRequestHelper.CreateSearchTemplate("artist", name, resultsCount ?? 1, 0)).ConfigureAwait(false);
var artistResult = await MusicBrainzRequestHelper.GetAsync<ArtistResult>(HttpClientFactory, MusicBrainzRequestHelper.CreateSearchTemplate("artist", name, resultsCount ?? 1, 0)).ConfigureAwait(false);
if (artistResult == null || artistResult.artists == null || !artistResult.artists.Any() || artistResult.count < 1)
{
return null;
}
var mbId = artistResult.artists.First().id;
// Now perform a detail request to get the details by the MbId
result = await MusicBrainzRequestHelper.GetAsync<Artist>(MusicBrainzRequestHelper.CreateLookupUrl("artist", mbId, "aliases+tags+genres+url-rels")).ConfigureAwait(false);
result = await MusicBrainzRequestHelper.GetAsync<Artist>(HttpClientFactory, MusicBrainzRequestHelper.CreateLookupUrl("artist", mbId, "aliases+tags+genres+url-rels")).ConfigureAwait(false);
if (result != null)
{
col.Insert(new RepositoryArtist
@ -112,7 +119,7 @@ namespace Roadie.Library.MetaData.MusicBrainz
var pageSize = 50;
var page = 0;
var url = MusicBrainzRequestHelper.CreateArtistBrowseTemplate(artistMbId, pageSize, 0);
var mbReleaseBrowseResult = await MusicBrainzRequestHelper.GetAsync<ReleaseBrowseResult>(url).ConfigureAwait(false);
var mbReleaseBrowseResult = await MusicBrainzRequestHelper.GetAsync<ReleaseBrowseResult>(HttpClientFactory, url).ConfigureAwait(false);
var totalReleases = mbReleaseBrowseResult != null ? mbReleaseBrowseResult.releasecount : 0;
var totalPages = Math.Ceiling((decimal)totalReleases / pageSize);
var fetchResult = new List<Release>();
@ -123,7 +130,7 @@ namespace Roadie.Library.MetaData.MusicBrainz
fetchResult.AddRange(mbReleaseBrowseResult.releases.Where(x => !string.IsNullOrEmpty(x.date)));
}
page++;
mbReleaseBrowseResult = await MusicBrainzRequestHelper.GetAsync<ReleaseBrowseResult>(MusicBrainzRequestHelper.CreateArtistBrowseTemplate(artistMbId, pageSize, pageSize * page)).ConfigureAwait(false);
mbReleaseBrowseResult = await MusicBrainzRequestHelper.GetAsync<ReleaseBrowseResult>(HttpClientFactory, MusicBrainzRequestHelper.CreateArtistBrowseTemplate(artistMbId, pageSize, pageSize * page)).ConfigureAwait(false);
} while (page < totalPages);
var releasesToInsert = fetchResult.GroupBy(x => x.title).Select(x => x.OrderBy(x => x.date).First()).OrderBy(x => x.date).ThenBy(x => x.title);
col.InsertBulk(releasesToInsert.Where(x => x != null).Select(x => new RepositoryRelease

34
Roadie.Api.Library/SearchEngines/MetaData/MusicBrainz/MusicBrainzRequestHelper.cs
View file

@ -2,6 +2,7 @@
using System;
using System.Diagnostics;
using System.Net;
using System.Net.Http;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
@ -11,28 +12,23 @@ namespace Roadie.Library.MetaData.MusicBrainz
public static class MusicBrainzRequestHelper
{
private const string LookupTemplate = "{0}/{1}/?inc={2}&fmt=json&limit=100";
private const int MaxRetries = 6;
private const string ReleaseBrowseTemplate = "release?artist={0}&limit={1}&offset={2}&fmt=json&inc={3}";
private const string SearchTemplate = "{0}?query={1}&limit={2}&offset={3}&fmt=json";
private const string WebServiceUrl = "http://musicbrainz.org/ws/2/";
internal static string CreateArtistBrowseTemplate(string id, int limit, int offset)
{
return string.Format("{0}{1}", WebServiceUrl, string.Format(ReleaseBrowseTemplate, id, limit, offset, "labels+aliases+recordings+release-groups+media+url-rels+tags+genres"));
}
internal static string CreateArtistBrowseTemplate(string id, int limit, int offset) => string.Format("{0}{1}", WebServiceUrl, string.Format(ReleaseBrowseTemplate, id, limit, offset, "labels+aliases+recordings+release-groups+media+url-rels+tags+genres"));
internal static string CreateCoverArtReleaseUrl(string musicBrainzId)
{
return string.Format("http://coverartarchive.org/release/{0}", musicBrainzId);
}
internal static string CreateCoverArtReleaseUrl(string musicBrainzId) => string.Format("http://coverartarchive.org/release/{0}", musicBrainzId);
/// <summary>
/// Creates a webservice lookup template.
/// </summary>
internal static string CreateLookupUrl(string entity, string mbid, string inc)
{
return string.Format("{0}{1}", WebServiceUrl, string.Format(LookupTemplate, entity, mbid, inc));
}
internal static string CreateLookupUrl(string entity, string mbid, string inc) => string.Format("{0}{1}", WebServiceUrl, string.Format(LookupTemplate, entity, mbid, inc));
/// <summary>
/// Creates a webservice search template.
@ -44,7 +40,7 @@ namespace Roadie.Library.MetaData.MusicBrainz
return string.Format("{0}{1}", WebServiceUrl, string.Format(SearchTemplate, entity, query, limit, offset));
}
internal static async Task<T> GetAsync<T>(string url, bool withoutMetadata = true)
internal static async Task<T> GetAsync<T>(IHttpClientFactory httpClientFactory, string url, bool withoutMetadata = true)
{
var tryCount = 0;
var result = default(T);
@ -53,16 +49,16 @@ namespace Roadie.Library.MetaData.MusicBrainz
{
try
{
using (var webClient = new WebClient())
{
webClient.Headers.Add("user-agent", WebHelper.UserAgent);
downloadedString = await webClient.DownloadStringTaskAsync(new Uri(url)).ConfigureAwait(false);
if (!string.IsNullOrWhiteSpace(downloadedString))
var client = httpClientFactory.CreateClient();
var request = new HttpRequestMessage(HttpMethod.Get, url);
request.Headers.Add("User-Agent", WebHelper.UserAgent);
var response = await client.SendAsync(request).ConfigureAwait(false);
if (response.IsSuccessStatusCode)
{
downloadedString = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
result = JsonSerializer.Deserialize<T>(downloadedString);
}
}
}
catch (WebException ex)
{
var response = ex.Response as HttpWebResponse;

58
Roadie.Api.Library/Utility/EncryptionHelper.cs
View file

@ -1,60 +1,46 @@
using System;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
namespace Roadie.Library.Utility
{
public static class EncryptionHelper
{
public static string Decrypt(string cyphertext, string key)
{
if (string.IsNullOrEmpty(cyphertext) || string.IsNullOrEmpty(key)) return null;
if (key.Length > 16) key = key.Substring(0, 16);
if (string.IsNullOrEmpty(cyphertext) || string.IsNullOrEmpty(key))
{
return null;
}
if (key.Length > 16)
{
key = key.Substring(0, 16);
}
return Decrypt(Convert.FromBase64String(cyphertext), System.Text.Encoding.UTF8.GetBytes(key));
}
public static string Decrypt(byte[] cyphertext, byte[] key)
public static string Decrypt(byte[] encryptedData, byte[] key)
{
using (var ms = new MemoryStream(cyphertext))
using (var desObj = Rijndael.Create())
{
desObj.Key = key;
var iv = new byte[16];
var offset = 0;
while (offset < iv.Length) offset += ms.Read(iv, offset, iv.Length - offset);
desObj.IV = iv;
using (var cs = new CryptoStream(ms, desObj.CreateDecryptor(), CryptoStreamMode.Read))
using (var sr = new StreamReader(cs, System.Text.Encoding.UTF8))
{
return sr.ReadToEnd();
}
}
return SymmetricEncryptor.DecryptToString(encryptedData, key);
}
public static string Encrypt(string plaintext, string key)
{
if (string.IsNullOrEmpty(plaintext) || string.IsNullOrEmpty(key)) return null;
if (key.Length > 16) key = key.Substring(0, 16);
if (string.IsNullOrEmpty(plaintext) || string.IsNullOrEmpty(key))
{
return null;
}
if (key.Length > 16)
{
key = key.Substring(0, 16);
}
return Convert.ToBase64String(Encrypt(plaintext, System.Text.Encoding.UTF8.GetBytes(key)));
}
public static byte[] Encrypt(string plaintext, byte[] key)
public static byte[] Encrypt(string toEncrypt, byte[] key)
{
using (var desObj = Rijndael.Create())
{
desObj.Key = key;
using (var ms = new MemoryStream())
{
ms.Write(desObj.IV, 0, desObj.IV.Length);
using (var cs = new CryptoStream(ms, desObj.CreateEncryptor(), CryptoStreamMode.Write))
{
var plainTextBytes = System.Text.Encoding.UTF8.GetBytes(plaintext);
cs.Write(plainTextBytes, 0, plainTextBytes.Length);
}
return ms.ToArray();
}
}
return SymmetricEncryptor.EncryptString(toEncrypt, key);
}
}
}

167
Roadie.Api.Library/Utility/SymmetricEncryptor.cs Normal file
View file

@ -0,0 +1,167 @@
using System;
using System.Linq;
using System.Security.Cryptography;
namespace Roadie.Library.Utility
{
/// <summary>
/// AES Encryption class, from https://tomrucki.com/posts/aes-encryption-in-csharp/
/// </summary>
public static class SymmetricEncryptor
{
private const int AesBlockByteSize = 128 / 8;
private const int PasswordSaltByteSize = 128 / 8;
private const int PasswordByteSize = 256 / 8;
private const int PasswordIterationCount = 100_000;
private const int SignatureByteSize = 256 / 8;
private const int MinimumEncryptedMessageByteSize =
PasswordSaltByteSize + // auth salt
PasswordSaltByteSize + // key salt
AesBlockByteSize + // IV
AesBlockByteSize + // cipher text min length
SignatureByteSize; // signature tag
private static readonly System.Text.Encoding StringEncoding = System.Text.Encoding.UTF8;
private static readonly RandomNumberGenerator Random = RandomNumberGenerator.Create();
public static byte[] EncryptString(string toEncrypt, byte[] password) => EncryptString(toEncrypt, System.Text.Encoding.UTF8.GetString(password));
public static byte[] EncryptString(string toEncrypt, string password)
{
// encrypt
var keySalt = GenerateRandomBytes(PasswordSaltByteSize);
var key = GetKey(password, keySalt);
var iv = GenerateRandomBytes(AesBlockByteSize);
byte[] cipherText;
using (var aes = CreateAes())
using (var encryptor = aes.CreateEncryptor(key, iv))
{
var plainText = StringEncoding.GetBytes(toEncrypt);
cipherText = encryptor
.TransformFinalBlock(plainText, 0, plainText.Length);
}
// sign
var authKeySalt = GenerateRandomBytes(PasswordSaltByteSize);
var authKey = GetKey(password, authKeySalt);
var result = MergeArrays(
additionalCapacity: SignatureByteSize,
authKeySalt, keySalt, iv, cipherText);
using (var hmac = new HMACSHA256(authKey))
{
var payloadToSignLength = result.Length - SignatureByteSize;
var signatureTag = hmac.ComputeHash(result, 0, payloadToSignLength);
signatureTag.CopyTo(result, payloadToSignLength);
}
return result;
}
public static string DecryptToString(byte[] encryptedData, byte[] password) => DecryptToString(encryptedData, System.Text.Encoding.UTF8.GetString(password));
public static string DecryptToString(byte[] encryptedData, string password)
{
if (encryptedData is null
|| encryptedData.Length < MinimumEncryptedMessageByteSize)
{
throw new ArgumentException("Invalid length of encrypted data");
}
var authKeySalt = encryptedData
.AsSpan(0, PasswordSaltByteSize).ToArray();
var keySalt = encryptedData
.AsSpan(PasswordSaltByteSize, PasswordSaltByteSize).ToArray();
var iv = encryptedData
.AsSpan(2 * PasswordSaltByteSize, AesBlockByteSize).ToArray();
var signatureTag = encryptedData
.AsSpan(encryptedData.Length - SignatureByteSize, SignatureByteSize).ToArray();
var cipherTextIndex = authKeySalt.Length + keySalt.Length + iv.Length;
var cipherTextLength =
encryptedData.Length - cipherTextIndex - signatureTag.Length;
var authKey = GetKey(password, authKeySalt);
var key = GetKey(password, keySalt);
// verify signature
using (var hmac = new HMACSHA256(authKey))
{
var payloadToSignLength = encryptedData.Length - SignatureByteSize;
var signatureTagExpected = hmac
.ComputeHash(encryptedData, 0, payloadToSignLength);
// constant time checking to prevent timing attacks
var signatureVerificationResult = 0;
for (int i = 0; i < signatureTag.Length; i++)
{
signatureVerificationResult |= signatureTag[i] ^ signatureTagExpected[i];
}
if (signatureVerificationResult != 0)
{
throw new CryptographicException("Invalid signature");
}
}
// decrypt
using (var aes = CreateAes())
{
using (var encryptor = aes.CreateDecryptor(key, iv))
{
var decryptedBytes = encryptor
.TransformFinalBlock(encryptedData, cipherTextIndex, cipherTextLength);
return StringEncoding.GetString(decryptedBytes);
}
}
}
private static Aes CreateAes()
{
var aes = Aes.Create();
aes.Mode = CipherMode.CBC;
aes.Padding = PaddingMode.PKCS7;
return aes;
}
private static byte[] GetKey(string password, byte[] passwordSalt)
{
var keyBytes = StringEncoding.GetBytes(password);
using (var derivator = new Rfc2898DeriveBytes(
keyBytes, passwordSalt,
PasswordIterationCount, HashAlgorithmName.SHA256))
{
return derivator.GetBytes(PasswordByteSize);
}
}
private static byte[] GenerateRandomBytes(int numberOfBytes)
{
var randomBytes = new byte[numberOfBytes];
Random.GetBytes(randomBytes);
return randomBytes;
}
private static byte[] MergeArrays(int additionalCapacity = 0, params byte[][] arrays)
{
var merged = new byte[arrays.Sum(a => a.Length) + additionalCapacity];
var mergeIndex = 0;
for (int i = 0; i < arrays.GetLength(0); i++)
{
arrays[i].CopyTo(merged, mergeIndex);
mergeIndex += arrays[i].Length;
}
return merged;
}
}
}

4
Roadie.Api.Services/AdminService.cs
View file

@ -658,9 +658,7 @@ namespace Roadie.Api.Services
});
}
/// <summary>
/// Perform checks/setup on start of application
/// </summary>
public void PerformStartUpTasks()
{
var sw = Stopwatch.StartNew();

3
Roadie.Api.Services/IAdminService.cs
View file

@ -31,6 +31,9 @@ namespace Roadie.Api.Services
Task<OperationResult<Dictionary<string, List<string>>>> MissingCollectionReleasesAsync(User user);
/// <summary>
/// Perform checks/setup on start of application
/// </summary>
void PerformStartUpTasks();
Task<OperationResult<bool>> ScanAllCollectionsAsync(User user, bool isReadOnly = false, bool doPurgeFirst = false);

6
Roadie.Api/Controllers/AccountController.cs
View file

@ -17,7 +17,6 @@ using System.Linq;
using System.Net;
using System.Text;
using System.Text.Encodings.Web;
using System.Text.Json;
using System.Threading.Tasks;
namespace Roadie.Api.Controllers
@ -28,7 +27,6 @@ namespace Roadie.Api.Controllers
[AllowAnonymous]
public class AccountController : ControllerBase
{
private IAdminService AdminService { get; }
private string BaseUrl
@ -65,9 +63,13 @@ namespace Roadie.Api.Controllers
private IRoadieSettings RoadieSettings { get; }
private string _baseUrl;
private readonly ILogger<AccountController> Logger;
private readonly SignInManager<User> SignInManager;
private readonly ITokenService TokenService;
private readonly UserManager<User> UserManager;
public AccountController(

1
Roadie.Api/Controllers/EntityControllerBase.cs
View file

@ -12,7 +12,6 @@ using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Net;
using System.Text.Json;
using System.Threading.Tasks;
using models = Roadie.Library.Models.Users;

1
Roadie.Api/Controllers/ImageController.cs
View file

@ -15,7 +15,6 @@ namespace Roadie.Api.Controllers
[Produces("application/json")]
[Route("images")]
[ApiController]
// [Authorize]
public class ImageController : EntityControllerBase
{
private IImageService ImageService { get; }

1
Roadie.Api/Controllers/SubsonicController.cs
View file

@ -14,7 +14,6 @@ using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Text.Json;
using System.Threading.Tasks;
using User = Roadie.Library.Models.Users.User;

4
Roadie.Api/Controllers/SystemController.cs
View file

@ -2,13 +2,9 @@
using Microsoft.AspNetCore.Identity;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Roadie.Api.Services;
using Roadie.Library.Caching;
using Roadie.Library.Configuration;
using Roadie.Library.Identity;
using System.Collections.Generic;
using System.Diagnostics;
using System.Threading.Tasks;
namespace Roadie.Api.Controllers
{

1
Roadie.Api/Controllers/UserController.cs
View file

@ -22,6 +22,7 @@ namespace Roadie.Api.Controllers
public class UserController : EntityControllerBase
{
private IHttpContext RoadieHttpContext { get; }
private IUserService UserService { get; }
private readonly ITokenService TokenService;

1
Roadie.Api/LoggingTraceListener.cs
View file

@ -1,5 +1,4 @@
using Serilog;
using System;
using System.Diagnostics;
namespace Roadie.Api

4
Roadie.Api/ModelBinding/SubsonicRequestBinder.cs
View file

@ -10,9 +10,9 @@ using System.Threading.Tasks;
namespace Roadie.Api.ModelBinding
{
/// <summary>
/// This is needed as some clienst post some get, some query string some body post.
/// This is needed as some clients post some get, some query string some body post.
/// </summary>
class SubsonicRequestBinder : IModelBinder
internal class SubsonicRequestBinder : IModelBinder
{
public Task BindModelAsync(ModelBindingContext bindingContext)
{

1
Roadie.Api/Models/LoginModel.cs
View file

@ -5,6 +5,7 @@ namespace Roadie.Api.Models
public class LoginModel
{
[Required] public string Password { get; set; }
[Required] public string Username { get; set; }
}
}

2
Roadie.Api/Models/ResetPasswordModel.cs
View file

@ -4,13 +4,11 @@ namespace Roadie.Api.Models
{
public class ResetPasswordModel : LoginModel
{
[Required]
[Compare(nameof(Password))]
public string PasswordConfirmation { get; set; }
[Required]
public string Token { get; set; }
}
}

17
Roadie.Api/Startup.cs
View file

@ -16,7 +16,6 @@ using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.IdentityModel.Tokens;
using Polly;
using Roadie.Api.Hubs;
using Roadie.Api.ModelBinding;
using Roadie.Api.Services;
@ -46,7 +45,6 @@ using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Net.Http;
using System.Reflection;
using System.Text;
@ -64,7 +62,7 @@ namespace Roadie.Api
TypeAdapterConfig.GlobalSettings.Default.PreserveReference(true);
}
public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
public void Configure(IApplicationBuilder app, IWebHostEnvironment env, IAdminService adminService)
{
if (env.IsDevelopment())
{
@ -102,6 +100,8 @@ namespace Roadie.Api
endpoints.MapControllers();
endpoints.MapDefaultControllerRoute();
});
adminService.PerformStartUpTasks();
}
// This method gets called by the runtime. Use this method to add services to the container.
@ -117,7 +117,7 @@ namespace Roadie.Api
services.AddSingleton<ICacheSerializer>(options =>
{
var logger = options.GetService<ILogger<Utf8JsonCacheSerializer>>();
return new Utf8JsonCacheSerializer(logger);
return new SystemTextCacheSerializer(logger);
});
services.AddSingleton<ICacheManager>(options =>
@ -335,7 +335,7 @@ namespace Roadie.Api
options.RespectBrowserAcceptHeader = true; // false by default
options.ModelBinderProviders.Insert(0, new SubsonicRequestBinderProvider());
})
.AddJsonOptions(options => options.JsonSerializerOptions.IgnoreNullValues = true)
.AddJsonOptions(options => options.JsonSerializerOptions.DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull)
.AddXmlSerializerFormatters();
services.Configure<IdentityOptions>(options =>
@ -362,9 +362,6 @@ namespace Roadie.Api
return new HttpContext(factory.GetService<IRoadieSettings>(), new UrlHelper(actionContext));
});
var sp = services.BuildServiceProvider();
var adminService = sp.GetService<IAdminService>();
adminService.PerformStartUpTasks();
}
private static string _roadieApiVersion = null;
@ -385,9 +382,13 @@ namespace Roadie.Api
private class IntegrationKey
{
public string BingImageSearch { get; set; }
public string DiscogsConsumerKey { get; set; }
public string DiscogsConsumerSecret { get; set; }
public string LastFMApiKey { get; set; }
public string LastFMSecret { get; set; }
}
}