Add S3 functionality

This commit is contained in:
An Phan 2016-06-13 17:04:42 +08:00
parent ad150daa5e
commit c098301167
No known key found for this signature in database
GPG key ID: 05536BB4BCDC02A2
22 changed files with 665 additions and 6 deletions

View file

@ -2,6 +2,11 @@ APP_ENV=production
APP_DEBUG=true
APP_URL=http://localhost
# If you want to use Amazon S3 with Koel, fill the info here and follow the
# installation guide at https://github.com/phanan/koel-aws.
AWS_ACCESS_KEY_ID=
AWS_SECRET_ACCESS_KEY=
AWS_REGION=
# Database connection name, which corresponds to the database driver.
# Possible values are:
@ -34,7 +39,7 @@ ADMIN_PASSWORD=
APP_MAX_SCAN_TIME=600
# The streaming method.
# The streaming method. Only used if STORAGE_DRIVER is "local".
# Can be either 'php' (default), 'x-sendfile', or 'x-accel-redirect'
# See https://github.com/phanan/koel/wiki#streaming-music for more information.
# Note: This setting doesn't have effect if the media needs transcoding (e.g. FLAC).

View file

@ -0,0 +1,9 @@
<?php
namespace App\Http\Controllers\API\ObjectStorage;
use App\Http\Controllers\API\Controller as BaseController;
class Controller extends BaseController
{
}

View file

@ -0,0 +1,9 @@
<?php
namespace App\Http\Controllers\API\ObjectStorage\S3;
use App\Http\Controllers\API\ObjectStorage\Controller as BaseController;
class Controller extends BaseController
{
}

View file

@ -0,0 +1,65 @@
<?php
namespace App\Http\Controllers\API\ObjectStorage\S3;
use App\Events\LibraryChanged;
use App\Http\Requests\API\ObjectStorage\S3\PutSongRequest;
use App\Http\Requests\API\ObjectStorage\S3\RemoveSongRequest;
use App\Models\Album;
use App\Models\Artist;
use App\Models\Song;
use Media;
class SongController extends Controller
{
/**
* Store a new song or update an existing one with data from AWS.
*
* @param PutSongRequest $request
*
* @return \Illuminate\Http\JsonResponse
*/
public function put(PutSongRequest $request)
{
$path = "s3://{$request->bucket}/{$request->key}";
$tags = $request->tags;
$artist = Artist::get(array_get($tags, 'artist'));
$compilation = (bool) trim(array_get($tags, 'albumartist'));
$album = Album::get($artist, array_get($tags, 'album'), $compilation);
if ($cover = array_get($tags, 'cover')) {
$album->writeCoverFile(base64_decode($cover['data']), $cover['extension']);
}
$song = Song::updateOrCreate(['id' => Media::getHash($path)], [
'path' => $path,
'album_id' => $album->id,
'contributing_artist_id' => $compilation ? $artist->id : null,
'title' => trim(array_get($tags, 'title', '')),
'length' => array_get($tags, 'duration', 0),
'track' => intval(array_get($tags, 'track')),
'lyrics' => array_get($tags, 'lyrics', ''),
'mtime' => time(),
]);
return response()->json($song);
}
/**
* Remove a song whose info matches with data sent from AWS.
*
* @param RemoveSongRequest $request
*
* @return \Illuminate\Http\JsonResponse
*/
public function remove(RemoveSongRequest $request)
{
abort_unless($song = Song::byPath("s3://{$request->bucket}/{$request->key}"), 404);
$song->delete();
event(new LibraryChanged());
return response()->json();
}
}

View file

@ -4,6 +4,7 @@ namespace App\Http\Controllers\API;
use App\Http\Requests\API\SongUpdateRequest;
use App\Http\Streamers\PHPStreamer;
use App\Http\Streamers\S3Streamer;
use App\Http\Streamers\TranscodingStreamer;
use App\Http\Streamers\XAccelRedirectStreamer;
use App\Http\Streamers\XSendFileStreamer;
@ -24,6 +25,10 @@ class SongController extends Controller
$bitrate = env('OUTPUT_BIT_RATE', 128);
}
if ($song->s3_params) {
return (new S3Streamer($song))->stream();
}
// If transcode parameter isn't passed, the default is to only transcode flac
if (is_null($transcode) && ends_with(mime_content_type($song->path), 'flac')) {
$transcode = true;

View file

@ -4,6 +4,7 @@ namespace App\Http;
use App\Http\Middleware\Authenticate;
use App\Http\Middleware\GetUserFromToken;
use App\Http\Middleware\ObjectStorageAuthenticate;
use Illuminate\Foundation\Http\Kernel as HttpKernel;
use Illuminate\Foundation\Http\Middleware\CheckForMaintenanceMode;
@ -26,5 +27,6 @@ class Kernel extends HttpKernel
protected $routeMiddleware = [
'auth' => Authenticate::class,
'jwt.auth' => GetUserFromToken::class,
'os.auth' => ObjectStorageAuthenticate::class,
];
}

View file

@ -0,0 +1,28 @@
<?php
namespace App\Http\Middleware;
use Closure;
/**
* Authenticate requests from Object Storage services (like S3).
* Such requests must have an apKey data, which matches with our app key.
*/
class ObjectStorageAuthenticate
{
/**
* Handle an incoming request.
*
* @param \Illuminate\Http\Request $request
* @param \Closure $next
* @return mixed
*/
public function handle($request, Closure $next)
{
if ($request->appKey !== config('app.key')) {
return response('Unauthorized.', 401);
}
return $next($request);
}
}

View file

@ -0,0 +1,26 @@
<?php
namespace App\Http\Requests\API\ObjectStorage;
use App\Http\Requests\API\Request as BaseRequest;
class Request extends BaseRequest
{
/**
* Determine if the user is authorized to make this request.
*
* @return bool
*/
public function authorize()
{
return true;
}
public function rules()
{
return [
'bucket' => 'required',
'key' => 'required',
];
}
}

View file

@ -0,0 +1,17 @@
<?php
namespace App\Http\Requests\API\ObjectStorage\S3;
use App\Http\Requests\API\ObjectStorage\S3\Request as BaseRequest;
class PutSongRequest extends BaseRequest
{
public function rules()
{
return [
'bucket' => 'required',
'key' => 'required',
'tags.duration' => 'required|numeric'
];
}
}

View file

@ -0,0 +1,16 @@
<?php
namespace App\Http\Requests\API\ObjectStorage\S3;
use App\Http\Requests\API\ObjectStorage\S3\Request as BaseRequest;
class RemoveSongRequest extends BaseRequest
{
public function rules()
{
return [
'bucket' => 'required',
'key' => 'required',
];
}
}

View file

@ -0,0 +1,9 @@
<?php
namespace App\Http\Requests\API\ObjectStorage\S3;
use App\Http\Requests\API\ObjectStorage\Request as BaseRequest;
class Request extends BaseRequest
{
}

View file

@ -0,0 +1,40 @@
<?php
namespace App\Http\Streamers;
use App\Models\Song;
use AWS;
use Aws\AwsClient;
class S3Streamer extends Streamer implements StreamerInterface
{
public function __construct(Song $song)
{
parent::__construct($song);
}
/**
* Stream the current song through S3.
* Actually, we only redirect to the S3 object's location.
*
* @param AwsClient $s3
*
* @return string
*/
public function stream(AwsClient $s3 = null)
{
if (!$s3) {
$s3 = AWS::createClient('s3');
}
$cmd = $s3->getCommand('GetObject', [
'Bucket' => $this->song->s3_params['bucket'],
'Key' => $this->song->s3_params['key'],
]);
$request = $s3->createPresignedRequest($cmd, '+1 hour');
// Get and redirect to the actual presigned-url
return redirect((string) $request->getUri());
}
}

View file

@ -25,7 +25,7 @@ class Streamer
{
$this->song = $song;
abort_unless(file_exists($this->song->path), 404);
abort_unless($this->song->s3_params || file_exists($this->song->path), 404);
// Hard code the content type instead of relying on PHP's fileinfo()
// or even Symfony's MIMETypeGuesser, since they appear to be wrong sometimes.

View file

@ -67,4 +67,11 @@ Route::group(['prefix' => 'api', 'namespace' => 'API'], function () {
Route::get('artist/{artist}/info', 'ArtistController@getInfo');
}
});
Route::group(['middleware' => 'os.auth', 'prefix' => 'os', 'namespace' => 'ObjectStorage'], function () {
Route::group(['prefix' => 's3', 'namespace' => 'S3'], function () {
Route::post('song', 'SongController@put'); // we follow AWS's convention here.
Route::delete('song', 'SongController@remove'); // and here.
});
});
});

View file

@ -118,7 +118,7 @@ class Album extends Model
* @param string $binaryData
* @param string $extension The file extension
*/
private function writeCoverFile($binaryData, $extension)
public function writeCoverFile($binaryData, $extension)
{
$extension = trim(strtolower($extension), '. ');
$fileName = uniqid().".$extension";

View file

@ -291,4 +291,30 @@ class Song extends Model
? $this->contributingArtist
: $this->album->artist;
}
/**
* Determine if the song is an AWS S3 Object.
*
* @return boolean
*/
public function isS3ObjectAttribute()
{
return strpos($this->path, 's3://') === 0;
}
/**
* Get the bucket and key name of an S3 object
*
* @return bool|array
*/
public function getS3ParamsAttribute()
{
if (!preg_match('/^s3:\\/\\/(.*)/', $this->path, $matches)) {
return false;
}
list($bucket, $key) = explode('/', $matches[1], 2);
return compact('bucket', 'key');
}
}

View file

@ -11,7 +11,8 @@
"phanan/cascading-config": "~2.0",
"barryvdh/laravel-ide-helper": "^2.1",
"guzzlehttp/guzzle": "^6.1",
"tymon/jwt-auth": "^0.5.6"
"tymon/jwt-auth": "^0.5.6",
"aws/aws-sdk-php-laravel": "^3.1"
},
"require-dev": {
"fzaninotto/faker": "~1.4",

195
composer.lock generated
View file

@ -4,9 +4,145 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file",
"This file is @generated automatically"
],
"hash": "73595742c708c946629155b84a000a62",
"content-hash": "b4898021e96521543995ae80b2a17399",
"hash": "2073946a21b9c8313b3f8953fc00937e",
"content-hash": "abdd95ff1703b935dc499162e82ba1a8",
"packages": [
{
"name": "aws/aws-sdk-php",
"version": "3.18.17",
"source": {
"type": "git",
"url": "https://github.com/aws/aws-sdk-php.git",
"reference": "6c7849556f556da8615d22e675710c7a086ed5d0"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/6c7849556f556da8615d22e675710c7a086ed5d0",
"reference": "6c7849556f556da8615d22e675710c7a086ed5d0",
"shasum": ""
},
"require": {
"guzzlehttp/guzzle": "~5.3|~6.0.1|~6.1",
"guzzlehttp/promises": "~1.0",
"guzzlehttp/psr7": "~1.0",
"mtdowling/jmespath.php": "~2.2",
"php": ">=5.5"
},
"require-dev": {
"andrewsville/php-token-reflection": "^1.4",
"aws/aws-php-sns-message-validator": "~1.0",
"behat/behat": "~3.0",
"doctrine/cache": "~1.4",
"ext-dom": "*",
"ext-json": "*",
"ext-openssl": "*",
"ext-pcre": "*",
"ext-simplexml": "*",
"ext-spl": "*",
"nette/neon": "^2.3",
"phpunit/phpunit": "~4.0|~5.0",
"psr/cache": "^1.0"
},
"suggest": {
"aws/aws-php-sns-message-validator": "To validate incoming SNS notifications",
"doctrine/cache": "To use the DoctrineCacheAdapter",
"ext-curl": "To send requests using cURL",
"ext-openssl": "Allows working with CloudFront private distributions and verifying received SNS messages"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "3.0-dev"
}
},
"autoload": {
"psr-4": {
"Aws\\": "src/"
},
"files": [
"src/functions.php"
]
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"Apache-2.0"
],
"authors": [
{
"name": "Amazon Web Services",
"homepage": "http://aws.amazon.com"
}
],
"description": "AWS SDK for PHP - Use Amazon Web Services in your PHP project",
"homepage": "http://aws.amazon.com/sdkforphp",
"keywords": [
"amazon",
"aws",
"cloud",
"dynamodb",
"ec2",
"glacier",
"s3",
"sdk"
],
"time": "2016-06-09 23:39:33"
},
{
"name": "aws/aws-sdk-php-laravel",
"version": "3.1.0",
"source": {
"type": "git",
"url": "https://github.com/aws/aws-sdk-php-laravel.git",
"reference": "3b946892d493b91b4920ec4facc4a0ad7195fb86"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/aws/aws-sdk-php-laravel/zipball/3b946892d493b91b4920ec4facc4a0ad7195fb86",
"reference": "3b946892d493b91b4920ec4facc4a0ad7195fb86",
"shasum": ""
},
"require": {
"aws/aws-sdk-php": "~3.0",
"illuminate/support": "~5.1",
"php": ">=5.5.9"
},
"require-dev": {
"phpunit/phpunit": "~4.0|~5.0"
},
"suggest": {
"laravel/framework": "To test the Laravel bindings",
"laravel/lumen-framework": "To test the Lumen bindings"
},
"type": "library",
"autoload": {
"psr-4": {
"Aws\\Laravel\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"Apache-2.0"
],
"authors": [
{
"name": "Amazon Web Services",
"homepage": "http://aws.amazon.com"
}
],
"description": "A simple Laravel 5 service provider for including the AWS SDK for PHP.",
"homepage": "http://aws.amazon.com/sdkforphp2",
"keywords": [
"amazon",
"aws",
"dynamodb",
"ec2",
"laravel",
"laravel 5",
"s3",
"sdk"
],
"time": "2016-01-18 06:57:07"
},
{
"name": "barryvdh/laravel-ide-helper",
"version": "v2.1.4",
@ -909,6 +1045,61 @@
],
"time": "2016-01-26 21:23:30"
},
{
"name": "mtdowling/jmespath.php",
"version": "2.3.0",
"source": {
"type": "git",
"url": "https://github.com/jmespath/jmespath.php.git",
"reference": "192f93e43c2c97acde7694993ab171b3de284093"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/jmespath/jmespath.php/zipball/192f93e43c2c97acde7694993ab171b3de284093",
"reference": "192f93e43c2c97acde7694993ab171b3de284093",
"shasum": ""
},
"require": {
"php": ">=5.4.0"
},
"require-dev": {
"phpunit/phpunit": "~4.0"
},
"bin": [
"bin/jp.php"
],
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "2.0-dev"
}
},
"autoload": {
"psr-4": {
"JmesPath\\": "src/"
},
"files": [
"src/JmesPath.php"
]
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Michael Dowling",
"email": "mtdowling@gmail.com",
"homepage": "https://github.com/mtdowling"
}
],
"description": "Declaratively specify how to extract elements from a JSON document",
"keywords": [
"json",
"jsonpath"
],
"time": "2016-01-05 18:25:05"
},
{
"name": "namshi/jose",
"version": "5.0.2",

View file

@ -141,6 +141,7 @@ return [
PhanAn\CascadingConfig\CascadingConfigServiceProvider::class,
Barryvdh\LaravelIdeHelper\IdeHelperServiceProvider::class,
Tymon\JWTAuth\Providers\JWTAuthServiceProvider::class,
Aws\Laravel\AwsServiceProvider::class,
/*
* Application Service Providers...
@ -206,6 +207,7 @@ return [
'Download' => App\Facades\Download::class,
'JWTAuth' => Tymon\JWTAuth\Facades\JWTAuth::class,
'JWTFactory' => Tymon\JWTAuth\Facades\JWTFactory::class,
'AWS' => Aws\Laravel\AwsFacade::class,
],

29
config/aws.php Normal file
View file

@ -0,0 +1,29 @@
<?php
use Aws\Laravel\AwsServiceProvider;
return [
/*
|--------------------------------------------------------------------------
| AWS SDK Configuration
|--------------------------------------------------------------------------
|
| The configuration options set in this file will be passed directly to the
| `Aws\Sdk` object, from which all client objects are created. The minimum
| required options are declared here, but the full set of possible options
| are documented at:
| http://docs.aws.amazon.com/aws-sdk-php/v3/guide/guide/configuration.html
|
*/
'credentials' => [
'key' => env('AWS_ACCESS_KEY_ID'),
'secret' => env('AWS_SECRET_ACCESS_KEY'),
],
'region' => env('AWS_REGION', 'us-east-1'),
'version' => 'latest',
'ua_append' => [
'L5MOD/' . AwsServiceProvider::VERSION,
],
];

119
config/ide-helper.php Normal file
View file

@ -0,0 +1,119 @@
<?php
return array(
/*
|--------------------------------------------------------------------------
| Filename & Format
|--------------------------------------------------------------------------
|
| The default filename (without extension) and the format (php or json)
|
*/
'filename' => '_ide_helper',
'format' => 'php',
/*
|--------------------------------------------------------------------------
| Helper files to include
|--------------------------------------------------------------------------
|
| Include helper files. By default not included, but can be toggled with the
| -- helpers (-H) option. Extra helper files can be included.
|
*/
'include_helpers' => false,
'helper_files' => array(
base_path().'/vendor/laravel/framework/src/Illuminate/Support/helpers.php',
),
/*
|--------------------------------------------------------------------------
| Model locations to include
|--------------------------------------------------------------------------
|
| Define in which directories the ide-helper:models command should look
| for models.
|
*/
'model_locations' => array(
'app',
),
/*
|--------------------------------------------------------------------------
| Extra classes
|--------------------------------------------------------------------------
|
| These implementations are not really extended, but called with magic functions
|
*/
'extra' => array(
'Eloquent' => array('Illuminate\Database\Eloquent\Builder', 'Illuminate\Database\Query\Builder'),
'Session' => array('Illuminate\Session\Store'),
),
'magic' => array(
'Log' => array(
'debug' => 'Monolog\Logger::addDebug',
'info' => 'Monolog\Logger::addInfo',
'notice' => 'Monolog\Logger::addNotice',
'warning' => 'Monolog\Logger::addWarning',
'error' => 'Monolog\Logger::addError',
'critical' => 'Monolog\Logger::addCritical',
'alert' => 'Monolog\Logger::addAlert',
'emergency' => 'Monolog\Logger::addEmergency',
)
),
/*
|--------------------------------------------------------------------------
| Interface implementations
|--------------------------------------------------------------------------
|
| These interfaces will be replaced with the implementing class. Some interfaces
| are detected by the helpers, others can be listed below.
|
*/
'interfaces' => array(
),
/*
|--------------------------------------------------------------------------
| Support for custom DB types
|--------------------------------------------------------------------------
|
| This setting allow you to map any custom database type (that you may have
| created using CREATE TYPE statement or imported using database plugin
| / extension to a Doctrine type.
|
| Each key in this array is a name of the Doctrine2 DBAL Platform. Currently valid names are:
| 'postgresql', 'db2', 'drizzle', 'mysql', 'oracle', 'sqlanywhere', 'sqlite', 'mssql'
|
| This name is returned by getName() method of the specific Doctrine/DBAL/Platforms/AbstractPlatform descendant
|
| The value of the array is an array of type mappings. Key is the name of the custom type,
| (for example, "jsonb" from Postgres 9.4) and the value is the name of the corresponding Doctrine2 type (in
| our case it is 'json_array'. Doctrine types are listed here:
| http://doctrine-dbal.readthedocs.org/en/latest/reference/types.html
|
| So to support jsonb in your models when working with Postgres, just add the following entry to the array below:
|
| "postgresql" => array(
| "jsonb" => "json_array",
| ),
|
*/
'custom_db_types' => array(
),
);

View file

@ -0,0 +1,53 @@
<?php
use App\Events\LibraryChanged;
use App\Models\Album;
use App\Models\Artist;
use App\Models\Song;
use Illuminate\Foundation\Testing\DatabaseTransactions;
use Illuminate\Foundation\Testing\WithoutMiddleware;
class ObjectStorage_S3Test extends TestCase
{
use DatabaseTransactions, WithoutMiddleware;
public function setUp()
{
parent::setUp();
}
public function testPut()
{
$this->post('api/os/s3/song', [
'bucket' => 'koel',
'key' => 'sample.mp3',
'tags' => [
'title' => 'A Koel Song',
'album' => 'Koel Testing Vol. 1',
'artist' => 'Koel',
'lyrics' => "When you wake up, turn your radio on, and you'll hear this simple song",
'duration' => 10,
'track' => 5,
],
])->seeInDatabase('songs', ['path' => 's3://koel/sample.mp3']);
}
public function testRemove()
{
$this->expectsEvents(LibraryChanged::class);
$this->post('api/os/s3/song', [
'bucket' => 'koel',
'key' => 'sample.mp3',
'tags' => [
'lyrics' => '',
'duration' => 10,
],
])->seeInDatabase('songs', ['path' => 's3://koel/sample.mp3']);
$this->delete('api/os/s3/song', [
'bucket' => 'koel',
'key' => 'sample.mp3',
])->notSeeInDatabase('songs', ['path' => 's3://koel/sample.mp3']);
}
}