Mirrors/koel
2
0
Fork 0
mirror of https://github.com/koel/koel synced 2024-11-10 06:34:14 +00:00
Code Issues Projects Releases Packages Wiki Activity

fix(sync): properly ignore unchanged files

Browse source
This commit is contained in:
Phan An 2022-07-07 12:45:47 +02:00
parent 527d4a073c
commit 09f54d26d5
No known key found for this signature in database
GPG key ID: A81E4477F0BB6FDC
16 changed files with 116 additions and 175 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

43
app/Console/Commands/SyncCommand.php
View file

@ -13,36 +13,34 @@ class SyncCommand extends Command
{
protected $signature = 'koel:sync
{record? : A single watch record. Consult Wiki for more info.}
{--excludes= : The comma-separated tags to excludes from syncing}
{--ignore= : The comma-separated tags to ignore (exclude) from syncing}
{--force : Force re-syncing even unchanged files}';
protected $description = 'Sync songs found in configured directory against the database.';
private int $ignored = 0;
private int $invalid = 0;
private int $synced = 0;
private MediaSyncService $mediaSyncService;
private int $skippedCount = 0;
private int $invalidCount = 0;
private int $syncedCount = 0;
private ?ProgressBar $progressBar = null;
public function __construct(MediaSyncService $mediaSyncService)
public function __construct(private MediaSyncService $mediaSyncService)
{
parent::__construct();
$this->mediaSyncService = $mediaSyncService;
}
public function handle(): void
public function handle(): int
{
$this->ensureMediaPath();
$record = $this->argument('record');
if (!$record) {
if ($record) {
$this->syncSingleRecord($record);
} else {
$this->syncAll();
return;
}
$this->syncSingleRecord($record);
return Command::SUCCESS;
}
/**
@ -50,20 +48,21 @@ class SyncCommand extends Command
*/
protected function syncAll(): void
{
$this->info('Syncing media from ' . Setting::get('media_path') . PHP_EOL);
$path = Setting::get('media_path');
$this->info('Syncing media from ' . $path . PHP_EOL);
// The excluded tags.
// Notice that this is only meaningful for existing records.
// New records will have every applicable field synced in.
$excludes = $this->option('excludes') ? explode(',', $this->option('excludes')) : [];
$this->mediaSyncService->sync(null, $excludes, $this->option('force'), $this);
$this->mediaSyncService->sync($excludes, $this->option('force'), $this);
$this->output->writeln(
PHP_EOL . PHP_EOL
. "<info>Completed! $this->synced new or updated song(s)</info>, "
. "$this->ignored unchanged song(s), "
. "and <comment>$this->invalid invalid file(s)</comment>."
. "<info>Completed! $this->syncedCount new or updated song(s)</info>, "
. "$this->skippedCount unchanged song(s), "
. "and <comment>$this->invalidCount invalid file(s)</comment>."
);
}
@ -90,15 +89,15 @@ class SyncCommand extends Command
$name = basename($path);
if ($result === FileSynchronizer::SYNC_RESULT_UNMODIFIED) {
++$this->ignored;
++$this->skippedCount;
} elseif ($result === FileSynchronizer::SYNC_RESULT_BAD_FILE) {
if ($this->option('verbose')) {
$this->error(PHP_EOL . "'$name' is not a valid media file: " . $reason);
$this->error(PHP_EOL . "'$name' is not a valid media file: $reason");
}
++$this->invalid;
++$this->invalidCount;
} else {
++$this->synced;
++$this->syncedCount;
}
}

4
app/Http/Controllers/API/ObjectStorage/S3/SongController.php
View file

@ -24,7 +24,7 @@ class SongController extends Controller
$request->key,
array_get($request->tags, 'artist'),
array_get($request->tags, 'album'),
(bool) trim(array_get($request->tags, 'albumartist')),
trim(array_get($request->tags, 'albumartist')),
array_get($request->tags, 'cover'),
trim(array_get($request->tags, 'title', '')),
(int) array_get($request->tags, 'duration', 0),
@ -39,7 +39,7 @@ class SongController extends Controller
{
try {
$this->s3Service->deleteSongEntry($request->bucket, $request->key);
} catch (SongPathNotFoundException $exception) {
} catch (SongPathNotFoundException) {
abort(Response::HTTP_NOT_FOUND);
}

5
app/Http/Controllers/API/SettingController.php
View file

@ -18,10 +18,7 @@ class SettingController extends Controller
public function update(SettingRequest $request)
{
Setting::set('media_path', rtrim(trim($request->media_path), '/'));
// In a next version we should opt for a "MediaPathChanged" event,
// but let's just do this async now.
$this->mediaSyncService->sync();
$this->mediaSyncService->sync(Setting::get('media_path'));
return response()->noContent();
}

9
app/Listeners/DeleteNonExistingRecordsPostSync.php
View file

@ -9,20 +9,15 @@ use App\Services\Helper;
class DeleteNonExistingRecordsPostSync
{
private SongRepository $songRepository;
private Helper $helper;
public function __construct(SongRepository $songRepository, Helper $helper)
public function __construct(private SongRepository $songRepository)
{
$this->songRepository = $songRepository;
$this->helper = $helper;
}
public function handle(MediaSyncCompleted $event): void
{
$hashes = $event->result
->validEntries()
->map(fn (string $path): string => $this->helper->getFileHash($path))
->map(static fn (string $path): string => Helper::getFileHash($path))
->merge($this->songRepository->getAllHostedOnS3()->pluck('id'))
->toArray();

1
app/Models/Album.php
View file

@ -38,7 +38,6 @@ use Laravel\Scout\Searchable;
* @method static Builder whereArtistIdAndName(int $id, string $name)
* @method static orderBy(...$params)
* @method static Builder latest()
* @method static Builder whereName(string $name)
*/
class Album extends Model
{

33
app/Models/Setting.php
View file

@ -2,6 +2,7 @@
namespace App\Models;
use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Database\Eloquent\Model;
@ -20,23 +21,18 @@ class Setting extends Model
public $timestamps = false;
protected $guarded = [];
/**
* Get a setting value.
*/
public static function get(string $key) // @phpcs:ignore
public static function get(string $key): mixed
{
$record = self::find($key);
return $record ? $record->value : null;
return self::find($key)?->value;
}
/**
* Set a setting (no pun) value.
*
* @param string|array $key the key of the setting, or an associative array of settings,
* @param array|string $key the key of the setting, or an associative array of settings,
* in which case $value will be discarded
*/
public static function set($key, $value = null): void
public static function set(array|string $key, $value = null): void
{
if (is_array($key)) {
foreach ($key as $k => $v) {
@ -49,20 +45,11 @@ class Setting extends Model
self::updateOrCreate(compact('key'), compact('value'));
}
/**
* Serialize the setting value before saving into the database.
* This makes settings more flexible.
*/
public function setValueAttribute($value): void
protected function value(): Attribute
{
$this->attributes['value'] = serialize($value);
}
/**
* Get the unserialized setting value.
*/
public function getValueAttribute($value) // @phpcs:ignore
{
return unserialize($value);
return new Attribute(
get: static fn ($value) => unserialize($value),
set: static fn ($value) => serialize($value)
);
}
}

2
app/Models/Song.php
View file

@ -38,6 +38,8 @@ use Laravel\Scout\Searchable;
* @method static Builder take(int $count)
* @method static float|int sum(string $column)
* @method static Builder latest(string $column = 'created_at')
* @method static Builder where(...$params)
* @method static Song findOrFail(string $id)
*/
class Song extends Model
{

14
app/Observers/SongObserver.php
View file

@ -7,20 +7,8 @@ use App\Services\Helper;
class SongObserver
{
private Helper $helper;
public function __construct(Helper $helper)
{
$this->helper = $helper;
}
public function creating(Song $song): void
{
$this->setFileHashAsId($song);
}
private function setFileHashAsId(Song $song): void
{
$song->id = $this->helper->getFileHash($song->path);
$song->id = Helper::getFileHash($song->path);
}
}

7
app/Repositories/SettingRepository.php
View file

@ -2,6 +2,8 @@
namespace App\Repositories;
use App\Models\Setting;
class SettingRepository extends AbstractRepository
{
/** @return array<mixed> */
@ -9,4 +11,9 @@ class SettingRepository extends AbstractRepository
{
return $this->model->pluck('value', 'key')->all();
}
public function getByKey(string $key): mixed
{
return Setting::get($key);
}
}

7
app/Repositories/SongRepository.php
View file

@ -30,14 +30,9 @@ class SongRepository extends AbstractRepository
private const VALID_SORT_COLUMNS = ['songs.title', 'songs.track', 'songs.length', 'artists.name', 'albums.name'];
private const DEFAULT_QUEUE_LIMIT = 500;
public function __construct(private Helper $helper)
{
parent::__construct();
}
public function getOneByPath(string $path): ?Song
{
return $this->getOneById($this->helper->getFileHash($path));
return $this->getOneById(Helper::getFileHash($path));
}
/** @return Collection|array<Song> */

29
app/Services/FileSynchronizer.php
View file

@ -39,7 +39,6 @@ class FileSynchronizer
public function __construct(
private getID3 $getID3,
private MediaMetadataService $mediaMetadataService,
private Helper $helper,
private SongRepository $songRepository,
private Cache $cache,
private Finder $finder
@ -48,51 +47,45 @@ class FileSynchronizer
public function setFile(string|SplFileInfo $path): self
{
$splFileInfo = $path instanceof SplFileInfo ? $path : new SplFileInfo($path);
$file = $path instanceof SplFileInfo ? $path : new SplFileInfo($path);
$this->filePath = $splFileInfo->getPathname();
$this->fileHash = $this->helper->getFileHash($this->filePath);
$this->filePath = $file->getPathname();
$this->fileHash = Helper::getFileHash($this->filePath);
$this->song = $this->songRepository->getOneById($this->fileHash); // @phpstan-ignore-line
$this->syncError = null;
$this->fileModifiedTime = Helper::getModifiedTime($file);
return $this;
}
public function getFileInfo(): ?SongScanInformation
public function getFileScanInformation(): ?SongScanInformation
{
$info = $this->getID3->analyze($this->filePath);
$this->syncError = Arr::get($info, 'error.0') ?: (Arr::get($info, 'playtime_seconds') ? null : 'Empty file');
if (isset($info['error']) || !isset($info['playtime_seconds'])) {
$this->syncError = isset($info['error']) ? $info['error'][0] : 'No playtime found';
return null;
}
return SongScanInformation::fromGetId3Info($info);
return $this->syncError ? null : SongScanInformation::fromGetId3Info($info);
}
/**
* Sync the song with all available media info into the database.
*
* @param array<string> $excludes The tags to exclude (only taken into account if the song already exists)
* @param array<string> $ignores The tags to ignore/exclude (only taken into account if the song already exists)
* @param bool $force Whether to force syncing, even if the file is unchanged
*/
public function sync(array $excludes = [], bool $force = false): int
public function sync(array $ignores = [], bool $force = false): int
{
if (!$this->isFileNewOrChanged() && !$force) {
return self::SYNC_RESULT_UNMODIFIED;
}
$info = $this->getFileInfo()?->toArray();
$info = $this->getFileScanInformation()?->toArray();
if (!$info) {
return self::SYNC_RESULT_BAD_FILE;
}
if (!$this->isFileNew()) {
foreach ($excludes as $tag) {
unset($info[$tag]);
}
Arr::forget($info, $ignores);
}
$artist = Arr::get($info, 'artist') ? Artist::getOrCreate($info['artist']) : $this->song->artist;

18
app/Services/Helper.php
View file

@ -2,14 +2,30 @@
namespace App\Services;
use SplFileInfo;
use Throwable;
class Helper
{
/**
* Get a unique hash from a file path.
* This hash can then be used as the Song record's ID.
*/
public function getFileHash(string $path): string
public static function getFileHash(string $path): string
{
return md5(config('app.key') . $path);
}
public static function getModifiedTime(string|SplFileInfo $file): int
{
$file = is_string($file) ? new SplFileInfo($file) : $file;
// Workaround for #344, where getMTime() fails for certain files with Unicode names on Windows.
try {
return $file->getMTime();
} catch (Throwable) {
// Just use current stamp for mtime.
return time();
}
}
}

20
app/Services/MediaSyncService.php
View file

@ -6,8 +6,8 @@ use App\Console\Commands\SyncCommand;
use App\Events\LibraryChanged;
use App\Events\MediaSyncCompleted;
use App\Libraries\WatchRecord\WatchRecordInterface;
use App\Models\Setting;
use App\Models\Song;
use App\Repositories\SettingRepository;
use App\Repositories\SongRepository;
use App\Values\SyncResult;
use Psr\Log\LoggerInterface;
@ -17,6 +17,7 @@ use Symfony\Component\Finder\Finder;
class MediaSyncService
{
public function __construct(
private SettingRepository $settingRepository,
private SongRepository $songRepository,
private FileSynchronizer $fileSynchronizer,
private Finder $finder,
@ -25,26 +26,25 @@ class MediaSyncService
}
/**
* @param array<string> $excludes The tags to exclude.
* @param array<string> $ignores The tags to ignore.
* Only taken into account for existing records.
* New records will have all tags synced in regardless.
* @param bool $force Whether to force syncing even unchanged files
* @param SyncCommand $syncCommand The SyncMedia command object, to log to console if executed by artisan
*/
public function sync(
?string $mediaPath = null,
array $excludes = [],
bool $force = false,
?SyncCommand $syncCommand = null
): void {
public function sync(array $ignores = [], bool $force = false, ?SyncCommand $syncCommand = null): void
{
/** @var string $mediaPath */
$mediaPath = $this->settingRepository->getByKey('media_path');
$this->setSystemRequirements();
$syncResult = SyncResult::init();
$songPaths = $this->gatherFiles($mediaPath ?: Setting::get('media_path'));
$songPaths = $this->gatherFiles($mediaPath);
$syncCommand?->createProgressBar(count($songPaths));
foreach ($songPaths as $path) {
$result = $this->fileSynchronizer->setFile($path)->sync($excludes, $force);
$result = $this->fileSynchronizer->setFile($path)->sync($ignores, $force);
switch ($result) {
case FileSynchronizer::SYNC_RESULT_SUCCESS:

35
app/Services/S3Service.php
View file

@ -13,24 +13,12 @@ use Illuminate\Cache\Repository as Cache;
class S3Service implements ObjectStorageInterface
{
private ?S3ClientInterface $s3Client;
private Cache $cache;
private MediaMetadataService $mediaMetadataService;
private SongRepository $songRepository;
private Helper $helper;
public function __construct(
?S3ClientInterface $s3Client,
Cache $cache,
MediaMetadataService $mediaMetadataService,
SongRepository $songRepository,
Helper $helper
private ?S3ClientInterface $s3Client,
private Cache $cache,
private MediaMetadataService $mediaMetadataService,
private SongRepository $songRepository,
) {
$this->s3Client = $s3Client;
$this->cache = $cache;
$this->mediaMetadataService = $mediaMetadataService;
$this->songRepository = $songRepository;
$this->helper = $helper;
}
public function getSongPublicUrl(Song $song): string
@ -53,7 +41,7 @@ class S3Service implements ObjectStorageInterface
string $key,
string $artistName,
string $albumName,
bool $compilation,
string $albumArtistName,
?array $cover,
string $title,
float $duration,
@ -63,7 +51,12 @@ class S3Service implements ObjectStorageInterface
$path = Song::getPathFromS3BucketAndKey($bucket, $key);
$artist = Artist::getOrCreate($artistName);
$album = Album::getOrCreate($artist, $albumName, $compilation);
$albumArtist = $albumArtistName && $albumArtistName !== $artistName
? Artist::getOrCreate($albumArtistName)
: $artist;
$album = Album::getOrCreate($albumArtist, $albumName);
if ($cover) {
$this->mediaMetadataService->writeAlbumCover(
@ -73,7 +66,7 @@ class S3Service implements ObjectStorageInterface
);
}
$song = Song::updateOrCreate(['id' => $this->helper->getFileHash($path)], [
$song = Song::updateOrCreate(['id' => Helper::getFileHash($path)], [
'path' => $path,
'album_id' => $album->id,
'artist_id' => $artist->id,
@ -94,9 +87,7 @@ class S3Service implements ObjectStorageInterface
$path = Song::getPathFromS3BucketAndKey($bucket, $key);
$song = $this->songRepository->getOneByPath($path);
if (!$song) {
throw SongPathNotFoundException::create($path);
}
throw_unless((bool) $song, SongPathNotFoundException::create($path));
$song->delete();
event(new LibraryChanged());

19
app/Services/Util.php
View file

@ -23,19 +23,14 @@ class Util
return 'UTF-16LE';
}
switch (substr($str, 0, 3)) {
case UTF8_BOM:
return 'UTF-8';
if (substr($str, 0, 3) === UTF8_BOM) {
return 'UTF-8';
}
switch (substr($str, 0, 4)) {
case UTF32_BIG_ENDIAN_BOM:
return 'UTF-32BE';
case UTF32_LITTLE_ENDIAN_BOM:
return 'UTF-32LE';
}
return null;
return match (substr($str, 0, 4)) {
UTF32_BIG_ENDIAN_BOM => 'UTF-32BE',
UTF32_LITTLE_ENDIAN_BOM => 'UTF-32LE',
default => null,
};
}
}

45
app/Values/SongScanInformation.php
View file

@ -4,10 +4,9 @@ namespace App\Values;
use App\Models\Album;
use App\Models\Artist;
use App\Services\Helper;
use Illuminate\Contracts\Support\Arrayable;
use Illuminate\Support\Arr;
use SplFileInfo;
use Throwable;
final class SongScanInformation implements Arrayable
{
@ -32,9 +31,6 @@ final class SongScanInformation implements Arrayable
$tags = array_merge(Arr::get($info, 'tags.id3v1', []), Arr::get($info, 'tags.id3v2', []));
$comments = Arr::get($info, 'comments', []);
$title = self::getTag($tags, 'title');
$albumName = self::getTag($tags, 'album', Album::UNKNOWN_NAME);
$artistName = self::getTag($tags, 'artist', Artist::UNKNOWN_NAME);
$albumArtistName = self::getTag($tags, ['albumartist', 'album_artist', 'band']);
// If the song is explicitly marked as a compilation but there's no album artist name, use the umbrella
@ -43,26 +39,20 @@ final class SongScanInformation implements Arrayable
$albumArtistName = Artist::VARIOUS_NAME;
}
$track = (int) self::getTag($tags, ['track', 'tracknumber', 'track_number']);
$disc = (int) self::getTag($tags, 'part_of_a_set', 1);
$lyrics = self::getTag($tags, ['unsynchronised_lyric', 'unsychronised_lyric']);
$path = Arr::get($info, 'filenamepath');
$length = (float) Arr::get($info, 'playtime_seconds');
$cover = self::getTag($comments, 'picture', []);
$mTime = self::getMTime($path);
return new self(
title: $title,
albumName: $albumName,
artistName: $artistName,
albumArtistName: $albumArtistName,
track: $track,
disc: $disc,
lyrics: $lyrics,
length: $length,
cover: $cover,
title: html_entity_decode(self::getTag($tags, 'title', pathinfo($path, PATHINFO_FILENAME))),
albumName: html_entity_decode(self::getTag($tags, 'album', Album::UNKNOWN_NAME)),
artistName: html_entity_decode(self::getTag($tags, 'artist', Artist::UNKNOWN_NAME)),
albumArtistName: html_entity_decode($albumArtistName),
track: (int) self::getTag($tags, ['track', 'tracknumber', 'track_number']),
disc: (int) self::getTag($tags, 'part_of_a_set', 1),
lyrics: html_entity_decode(self::getTag($tags, ['unsynchronised_lyric', 'unsychronised_lyric'])),
length: (float) Arr::get($info, 'playtime_seconds'),
cover: self::getTag($comments, 'picture', []),
path: $path,
mTime: $mTime,
mTime: Helper::getModifiedTime($path),
);
}
@ -81,19 +71,6 @@ final class SongScanInformation implements Arrayable
return $value ?? $default;
}
private static function getMTime(mixed $path): int
{
$splFileInfo = new SplFileInfo($path);
// Workaround for #344, where getMTime() fails for certain files with Unicode names on Windows.
try {
return $splFileInfo->getMTime();
} catch (Throwable) {
// Just use current stamp for mtime.
return time();
}
}
/** @return array<mixed> */
public function toArray(): array
{