From e514040e1ac1de4b6ad103c9e672b6401c59a848 Mon Sep 17 00:00:00 2001 From: Elliot Chance Date: Wed, 29 Dec 2021 00:54:57 -0500 Subject: [PATCH] Import from Apple Music (#25) Importing a tracklist from Apple Music is as easy as pasting the album URL into a new "Import From..." model. I have started with Apple Music, but I will add more soon. The Tracklist Editor still works entirely offline if you don't use the import feature. However, due to CORS the import feature cannot be part of the static web page. I will have to manually push changes to the importer. If that becomes a pain I can automate it with CD later. One known issue: If the album is semi-various artists, the importer will only catch a few tracks. Semi-various artists is a thing with Apple Music where albums that contain mostly one artist do not show that artist on those tracks. This causes havoc with the (probably brittle) regex that parses the raw HTML. --- import/.gitignore | 2 + import/.npmignore | 6 ++ import/Makefile | 8 ++ import/README.md | 22 ++++ import/apple-music.js | 72 +++++++++++++ import/apple-music.test.js | 200 +++++++++++++++++++++++++++++++++++++ import/package-lock.json | 71 +++++++++++++ import/package.json | 5 + import/serverless.yml | 22 ++++ index.html | 67 ++++++++++++- 10 files changed, 473 insertions(+), 2 deletions(-) create mode 100644 import/.gitignore create mode 100644 import/.npmignore create mode 100644 import/Makefile create mode 100644 import/README.md create mode 100644 import/apple-music.js create mode 100644 import/apple-music.test.js create mode 100644 import/package-lock.json create mode 100644 import/package.json create mode 100644 import/serverless.yml diff --git a/import/.gitignore b/import/.gitignore new file mode 100644 index 0000000..e0f039d --- /dev/null +++ b/import/.gitignore @@ -0,0 +1,2 @@ +.serverless +node_modules diff --git a/import/.npmignore b/import/.npmignore new file mode 100644 index 0000000..2b48c8b --- /dev/null +++ b/import/.npmignore @@ -0,0 +1,6 @@ +# package directories +node_modules +jspm_packages + +# Serverless directories +.serverless \ No newline at end of file diff --git a/import/Makefile b/import/Makefile new file mode 100644 index 0000000..b0e0cfb --- /dev/null +++ b/import/Makefile @@ -0,0 +1,8 @@ +test: + node apple-music.test.js + +dev: + sls deploy --stage dev + +prod: test + sls deploy --stage production diff --git a/import/README.md b/import/README.md new file mode 100644 index 0000000..27b15ad --- /dev/null +++ b/import/README.md @@ -0,0 +1,22 @@ +Some of these instructions are mostly for my own benefit, but if you wanted to +deploy your own copy for debugging or any other reason, go for it. + +# Run Tests + +```sh +make test +``` + +# Deploy + +Development: + +```bash +make dev +``` + +Production (will run tests first): + +```bash +make prod +``` diff --git a/import/apple-music.js b/import/apple-music.js new file mode 100644 index 0000000..5806c01 --- /dev/null +++ b/import/apple-music.js @@ -0,0 +1,72 @@ +'use strict'; + +// Examples: +// sls invoke -f apple-music -d '{"url":"https://music.apple.com/us/album/nye-2022-dj-mix/1600990821"}' +// curl "https://sxxqp2lco7.execute-api.us-east-1.amazonaws.com/dev/apple-music?url=https%3A%2F%2Fmusic.apple.com%2Fus%2Falbum%2Fnye-2022-dj-mix%2F1600990821" + +const fetch = require('node-fetch'); + +module.exports.import = async (event) => { + // Validate the URL to prevent mistakes and abuse. + const querystring = event.queryStringParameters; + const url = querystring.url; + if (!url.startsWith('https://music.apple.com/us/album/')) { + return response(400, { + error: `Invalid URL: ${url}`, + }); + } + + // Fetch the raw HTML. + const resp = await fetch(url); + const body = await resp.text(); + + // Parse tracks. + // + // TODO(elliotchance): This is an ultra crude regexp that will probably break + // in the future. + // + // There are a few layouts we need to test, specially various artist albums + // have a different layout than a single artist album. + + // If we get more than one match, this must be a VA album. The regexp happens + // to fall out that way, but also an album with one track could hardly be + // considered various artists. Maybe this is possible - I haven't seen any + // examples of this though. + let matches = Array.from(body.matchAll(/songs-list-row__song-name">([^<]+).*?row__link".*?>([^<]+).*?row__length">([^<]+)/gs)); + let decoder = (match, number) => ({ + number, + title: match[2].trim() + ' - ' + match[1].trim(), + time: match[3].trim(), + }); + + if (matches.length < 2) { + matches = Array.from(body.matchAll(/songs-list-row__song-name".*?>(.*?)<.*?row__length".*?>(.*?) ({ + number, + title: match[1].trim(), + time: match[2].trim(), + }); + } + + let tracks = []; + let number = 1; + for (const match of matches) { + tracks.push(decoder(match, number)); + ++number; + } + + return response(200, { + tracks, + }); +}; + +function response(statusCode, body) { + return { + statusCode, + body: JSON.stringify(body), + headers: { + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Credentials': true, + }, + }; +} diff --git a/import/apple-music.test.js b/import/apple-music.test.js new file mode 100644 index 0000000..9fa8c3f --- /dev/null +++ b/import/apple-music.test.js @@ -0,0 +1,200 @@ +const importer = require('./apple-music').import; +const assert = require('assert'); + +function assertAppleMusic(tt) { + (async () => { + const resp = JSON.parse((await importer({ + queryStringParameters: { + url: tt.url, + } + })).body); + + if (tt.error || resp.error) { + assert.equal(resp.error, tt.error); + return; + } + + assert.deepEqual(resp.tracks, tt.tracks); + })(); +} + +// Bad URL +assertAppleMusic({ + url: 'https://notapple.com/1600990821', + error: 'Invalid URL: https://notapple.com/1600990821', +}); + +// Single track +assertAppleMusic({ + url: 'https://music.apple.com/us/album/go-on-then-love-feat-the-maine-odea-remix-odea-remix-single/1591618917', + tracks: [ + { + number: 1, + title: 'Go On Then, Love (feat. The Maine) [Odea Remix]', + time: '3:08' + } + ], +}); + +// Single artist +assertAppleMusic({ + url: 'https://music.apple.com/us/album/dreaming-out-loud/1440889649', + tracks: [ + { number: 1, title: 'Say (All I Need)', time: '3:50' }, + { number: 2, title: 'Mercy', time: '4:01' }, + { number: 3, title: 'Stop and Stare', time: '3:44' }, + { number: 4, title: 'Apologize', time: '3:25' }, + { number: 5, title: 'Goodbye, Apathy', time: '3:32' }, + { number: 6, title: 'All Fall Down', time: '4:06' }, + { number: 7, title: 'Tyrant', time: '5:05' }, + { number: 8, title: 'Prodigal', time: '3:56' }, + { number: 9, title: "Won't Stop", time: '5:05' }, + { number: 10, title: 'All We Are', time: '4:23' }, + { number: 11, title: 'Someone to Save You', time: '4:13' }, + { number: 12, title: 'Come Home', time: '4:29' }, + { + number: 13, + title: 'Apologize (feat. OneRepublic)', + time: '3:05' + } + ], +}); + +// Various artists +assertAppleMusic({ + url: 'https://music.apple.com/us/album/nye-2022-dj-mix/1600990821', + tracks: [ + { + number: 1, + title: 'Westside Gunn - Praise God Intro (feat. AA Rashid) [Mixed]', + time: '2:03' + }, + { + number: 2, + title: "The S.S.O. Orchestra - Tonight's the Night (feat. Douglas Lucas & The Sugar Sisters) [Mixed]", + time: '1:11' + }, + { + number: 3, + title: 'Irish Coffee - The Show, Pt. 1 / Scrabble (Mixed)', + time: '1:42' + }, + { + number: 4, + title: "All The People - Cramp Your Style / Don't Play No Game That I Can't Win (feat. Santigold) [Remix] [Mixed]", + time: '1:21' + }, + { number: 5, title: 'Nas - Nasty (Mixed)', time: '1:01' }, + { + number: 6, + title: 'Johnny Pate - Shaft In Africa (Addis) [Mixed]', + time: '1:19' + }, + { + number: 7, + title: 'Freedom - Get Up and Dance (Mixed)', + time: '1:12' + }, + { + number: 8, + title: "Funky 4+1 - That's the Joint (Mixed)", + time: '1:00' + }, + { + number: 9, + title: 'Esther Williams - Last Night Changed It All (I Really Had a Ball) [Mixed]', + time: '1:06' + }, + { + number: 10, + title: "Jungle Brothers - Feelin' Alright / Love Vibration (Mixed)", + time: '0:51' + }, + { + number: 11, + title: 'Indeep - Last Night a D.J. Saved My Life / Agboju Logun (Mr Bongo 7" Edit) [Mixed]', + time: '1:50' + }, + { + number: 12, + title: 'Bobby Thurston - You Got What It Takes (Remix) [Mixed]', + time: '2:18' + }, + { + number: 13, + title: 'The Rebirth - Evil Vibrations (Remix) [Mixed]', + time: '1:33' + }, + { + number: 14, + title: 'Vaughan Mason and Crew - Bounce, Rock, Skate, Roll (Mixed)', + time: '3:24' + }, + { + number: 15, + title: 'Love Unlimited - I Did It for Love (Mixed)', + time: '1:59' + }, + { + number: 16, + title: 'King Sporty & The Root Rockers - Get on Down (Mixed)', + time: '1:20' + }, + { + number: 17, + title: 'El Turronero - Las Penas (Las Cañas) [Mixed]', + time: '2:06' + }, + { + number: 18, + title: "Sault - Don't Waste My Time (Mixed)", + time: '1:47' + }, + { + number: 19, + title: 'Can - Vitamin C (U.N.K.L.E. Mix) [Mixed]', + time: '3:24' + }, + { + number: 20, + title: 'Radiohead - Everything In Its Right Place (Mixed)', + time: '3:54' + }, + { + number: 21, + title: 'ID - ID1 (from NYE 2022: Zane Lowe) [Mixed]', + time: '2:47' + }, + { + number: 22, + title: 'Pete Le Freq - One More Step (Mixed)', + time: '3:17' + }, + { + number: 23, + title: 'ID - ID2 (from NYE 2022: Zane Lowe) [Mixed]', + time: '4:10' + }, + { number: 24, title: 'iZNiiK - Leave (Mixed)', time: '2:27' }, + { + number: 25, + title: 'Disclosure - In My Arms (Mixed)', + time: '2:09' + }, + { number: 26, title: 'SG Lewis - Time (Mixed)', time: '2:28' }, + { + number: 27, + title: 'Underworld - Two Months Off (Mixed)', + time: '5:48' + } + ], +}); + +// TODO(elliotchance): These ones do not working. + +// Various artists with mostly the same artist (so this they don't show on most +// of the tracks) +// assertAppleMusic({ +// url: 'https://music.apple.com/us/album/mirage/398383005', +// tracks: [], +// }); diff --git a/import/package-lock.json b/import/package-lock.json new file mode 100644 index 0000000..8a558ec --- /dev/null +++ b/import/package-lock.json @@ -0,0 +1,71 @@ +{ + "name": "import", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "dependencies": { + "node-fetch": "^2.6.6" + } + }, + "node_modules/node-fetch": { + "version": "2.6.6", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.6.tgz", + "integrity": "sha512-Z8/6vRlTUChSdIgMa51jxQ4lrw/Jy5SOW10ObaA47/RElsAN2c5Pn8bTgFGWn/ibwzXTE8qwr1Yzx28vsecXEA==", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + } + }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=" + }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE=" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha1-lmRU6HZUYuN2RNNib2dCzotwll0=", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + } + }, + "dependencies": { + "node-fetch": { + "version": "2.6.6", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.6.tgz", + "integrity": "sha512-Z8/6vRlTUChSdIgMa51jxQ4lrw/Jy5SOW10ObaA47/RElsAN2c5Pn8bTgFGWn/ibwzXTE8qwr1Yzx28vsecXEA==", + "requires": { + "whatwg-url": "^5.0.0" + } + }, + "tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=" + }, + "webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE=" + }, + "whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha1-lmRU6HZUYuN2RNNib2dCzotwll0=", + "requires": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + } + } +} diff --git a/import/package.json b/import/package.json new file mode 100644 index 0000000..336d935 --- /dev/null +++ b/import/package.json @@ -0,0 +1,5 @@ +{ + "dependencies": { + "node-fetch": "^2.6.6" + } +} diff --git a/import/serverless.yml b/import/serverless.yml new file mode 100644 index 0000000..e03fdd6 --- /dev/null +++ b/import/serverless.yml @@ -0,0 +1,22 @@ +service: import +app: tracklist-editor +org: elliotchance +frameworkVersion: '2' + +provider: + name: aws + runtime: nodejs12.x + lambdaHashingVersion: 20201221 + +functions: + apple-music: + handler: apple-music.import + events: + - http: + path: /apple-music + method: GET + cors: true + request: + parameters: + querystrings: + url: true diff --git a/index.html b/index.html index 4a9b0d7..1fb93db 100644 --- a/index.html +++ b/index.html @@ -26,6 +26,11 @@ }