diff --git a/README.md b/README.md index 05f22f170..f34080d7c 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,47 @@ During ideal conditions (Raspberry Pi 3 Model B+, class 10 SD card and fast inte Go through the steps in [this documentation](/docs/balena-fleet-deployment.md) to deploy Anthias on your own Balena fleet. +## Migrating assets from Anthias to Screenly + +This feature is only available in devices running Raspberry Pi OS at the moment. + +### Migrating from a device running Raspberry Pi OS (Lite) + +To get started, SSH to your Raspberry Pi running Anthias. For instance: + +```bash +ssh pi@raspberrypi +``` + +Go to the project root directory and create a Python virtual environment, if you haven't created one. + +```bash +cd ~/screenly +python -m venv venv/ +``` + +Activate the virtual environment. You need to do this everytime right before you run the script. + +```bash +source ./venv/bin/activate +``` + +Install the dependencies required by the assets migration script. + +```bash +pip install -r requirements/requirements.local.txt +``` + +Before running the script, you should prepare the following: +* Your Screenly API key +* Anthias username and password, if your device has basic authentication enabled + +Run the assets migration script. Follow through the instructions & prompts carefully. + +```bash +python tools/migrate-assets-to-screenly.py +``` + ## Issues and bugs Do however note that we are still in the process of knocking out some bugs. You can track the known issues [here](https://github.com/Screenly/Anthias/issues). You can also check the discussions in the [Anthias forums](https://forums.screenly.io). diff --git a/ansible/roles/screenly/tasks/main.yml b/ansible/roles/screenly/tasks/main.yml index 7c793da5e..1c6e64233 100644 --- a/ansible/roles/screenly/tasks/main.yml +++ b/ansible/roles/screenly/tasks/main.yml @@ -127,3 +127,10 @@ path: "/etc/systemd/system/{{ item }}" state: absent with_items: "{{ deprecated_screenly_systemd_units }}" + +- name: Remove the ngrok binary + ansible.builtin.file: + path: "/usr/local/bin/ngrok" + state: absent + owner: root + group: root diff --git a/ansible/roles/tools/files/ngrok b/ansible/roles/tools/files/ngrok deleted file mode 100755 index 0be6d0d10..000000000 Binary files a/ansible/roles/tools/files/ngrok and /dev/null differ diff --git a/ansible/roles/tools/tasks/main.yml b/ansible/roles/tools/tasks/main.yml deleted file mode 100644 index 39193768b..000000000 --- a/ansible/roles/tools/tasks/main.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -- name: Copy ngrok binary - ansible.builtin.copy: - src: ngrok - dest: /usr/local/bin/ - mode: "0755" - owner: root - group: root diff --git a/ansible/site.yml b/ansible/site.yml index 1e542f959..3d813d1d5 100644 --- a/ansible/site.yml +++ b/ansible/site.yml @@ -12,4 +12,3 @@ - screenly - network - splashscreen - - tools diff --git a/docker/nginx/nginx.conf b/docker/nginx/nginx.conf index 86127218b..22f3d96ec 100644 --- a/docker/nginx/nginx.conf +++ b/docker/nginx/nginx.conf @@ -86,15 +86,3 @@ server { alias /data/screenly/static; } } - -server { - # Only allow from localhost and Docker's CIDR - allow 172.16.0.0/12; - allow 172.0.0.1; - deny all; - - server_name *.ngrok.io; - listen 80; - root /data/screenly_assets; - try_files $uri /data/screenly_assets$uri; -} diff --git a/requirements/requirements.local.txt b/requirements/requirements.local.txt new file mode 100644 index 000000000..34a5fd7cb --- /dev/null +++ b/requirements/requirements.local.txt @@ -0,0 +1,5 @@ +# vim: ft=requirements + +click==8.1.7 +requests==2.32.3 +tenacity==8.4.1 diff --git a/tools/assets-migration-to-screenly-pro.py b/tools/assets-migration-to-screenly-pro.py deleted file mode 100644 index 923b6e0af..000000000 --- a/tools/assets-migration-to-screenly-pro.py +++ /dev/null @@ -1,218 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import unicode_literals -from builtins import str -import os -import sys -import traceback -import sh - -import click -import requests -import time - -from requests.auth import HTTPBasicAuth - -user_home_dir = os.getenv('HOME') - -BASE_API_SCREENLY_URL = 'https://api.screenlyapp.com' -ASSETS_SCREENLY_OSE_API = 'http://127.0.0.1/api/v1.1/assets' - -PORT_NGROK = 4040 -PORT = 80 - -token = None -ngrok_public_url = None - - -################################ -# Suprocesses -################################ - -def start_http_ngrok_process(try_connection=100): - click.echo(click.style("Ngrok starting ...", fg='yellow')) - - sh.ngrok('http', str(PORT), _bg=True, _in=os.devnull, _out=os.devnull, _err=sys.stderr) - - try_count = 0 - while True: - if try_count >= try_connection: - raise Exception('Failed start ngrok') - try: - requests.get('http://127.0.0.1:%i' % PORT_NGROK, timeout=10) - break - except requests.exceptions.ConnectionError: - try_count += 1 - time.sleep(0.1) - - click.echo(click.style("Ngrok successfull started", fg='green')) - - -def get_ngrock_public_url(try_connection=100): - try_count = 0 - while True: - if try_count >= try_connection: - raise Exception('Could not take a public url ngrok') - response = requests.get('http://127.0.0.1:%i/api/tunnels' % PORT_NGROK, timeout=10).json() - if response['tunnels']: - break - else: - try_count += 1 - time.sleep(0.1) - continue - return response['tunnels'][0]['public_url'] - - -################################ -# Utilities -################################ - -def progress_bar(count, total, text=''): - """ - This simple console progress bar - For display progress asset uploads - """ - progress_line = "\xe2" * int(round(50 * count / float(total))) + '-' * (50 - int(round(50 * count / float(total)))) - percent = round(100.0 * count / float(total), 1) - sys.stdout.write('[%s] %s%s %s\r' % (progress_line, percent, '%', text)) - sys.stdout.flush() - - -def set_token(value): - global token - token = 'Token %s' % value - - -def set_ngrok_public_url(value): - global ngrok_public_url - ngrok_public_url = value - - -################################ -# Database -################################ - -def get_assets_by_screenly_ose_api(): - if click.confirm('Do you need authentication to access Screenly-OSE API?'): - login = click.prompt('Login') - password = click.prompt('Password', hide_input=True) - auth = HTTPBasicAuth(login, password) - else: - auth = None - response = requests.get(ASSETS_SCREENLY_OSE_API, timeout=10, auth=auth) - if response.status_code == 200: - return response.json() - elif response.status_code == 401: - raise Exception('Access denied') - - -################################ -# Requests -################################ - -def send_asset(asset): - endpoind_url = '%s/api/v3/assets/' % BASE_API_SCREENLY_URL - headers = { - 'Authorization': token - } - asset_uri = asset['uri'] - if asset_uri.startswith(user_home_dir): - asset_uri = os.path.join(ngrok_public_url, asset['asset_id']) - data = { - 'title': asset['name'], - 'source_url': asset_uri - } - response = requests.post(endpoind_url, data=data, headers=headers) - return response.status_code == 200 - - -def check_validate_token(api_key): - endpoind_url = '%s/api/v3/assets/' % BASE_API_SCREENLY_URL - headers = { - 'Authorization': 'Token %s' % api_key - } - response = requests.get(endpoind_url, headers=headers) - if response.status_code == 200: - return api_key - else: - return None - - -def get_api_key_by_credentials(username, password): - endpoind_url = '%s/api/v3/tokens/' % BASE_API_SCREENLY_URL - data = { - 'username': username, - 'password': password - } - response = requests.post(endpoind_url, data=data) - if response.status_code == 200: - return response.json()['token'] - else: - return None - - -################################ -################################ - -def start_migration(): - if click.confirm('Do you want to start assets migration?'): - click.echo('\n') - start_http_ngrok_process() - set_ngrok_public_url(get_ngrock_public_url()) - click.echo('\n') - assets_migration() - - -def assets_migration(): - assets = get_assets_by_screenly_ose_api() - assets_length = len(assets) - click.echo('\n') - for index, asset in enumerate(assets): - asset_name = str(asset['name']) - progress_bar(index + 1, assets_length, text='Asset in migration progress: %s' % asset_name) - status = send_asset(asset) - if not status: - click.echo(click.style('\n%s asset was failed migration' % asset_name, fg='red')) - click.echo('\n') - click.echo(click.style('Migration completed successfully', fg='green')) - - -@click.command() -@click.option('--method', - prompt='What do you want to use for migration?\n1.API token\n2.Credentials\n0.Exit\nYour choice', - type=click.Choice(['1', '2', '0'])) -def main(method): - try: - valid_token = None - - if method == '1': - api_key = click.prompt('Your API key') - valid_token = check_validate_token(api_key) - elif method == '2': - username = click.prompt('Your username') - password = click.prompt('Your password', hide_input=True) - valid_token = get_api_key_by_credentials(username, password) - elif method == '0': - sys.exit(0) - - if valid_token: - set_token(valid_token) - click.echo(click.style('Successfull authentication', fg='green')) - start_migration() - else: - click.echo(click.style('Failed authentication', fg='red')) - except Exception: - traceback.print_exc() - - -if __name__ == '__main__': - click.echo(click.style(""" - _____ __ ____ _____ ______ - / ___/_____________ ___ ____ / /_ __ / __ \/ ___// ____/ - \__ \/ ___/ ___/ _ \/ _ \/ __ \/ / / / / / / / /\__ \/ __/ - ___/ / /__/ / / __/ __/ / / / / /_/ / / /_/ /___/ / /___ - /____/\___/_/ \___/\___/_/ /_/_/\__, / \____//____/_____/ - /____/ - """, fg='blue')) - - main() diff --git a/tools/migrate-assets-to-screenly.py b/tools/migrate-assets-to-screenly.py new file mode 100644 index 000000000..17c50d0f5 --- /dev/null +++ b/tools/migrate-assets-to-screenly.py @@ -0,0 +1,216 @@ +# -*- coding: utf-8 -*- + +import click +import os +import requests +import sys +import traceback + +from inspect import cleandoc +from requests.auth import HTTPBasicAuth +from requests.exceptions import RequestException +from tenacity import retry +from textwrap import shorten + +HOME = os.getenv('HOME') +BASE_API_SCREENLY_URL = 'https://api.screenlyapp.com' +ASSETS_ANTHIAS_API = 'http://127.0.0.1/api/v1.1/assets' +MAX_ASSET_NAME_LENGTH = 40 +PORT = 80 + +token = None + + +############# +# Utilities # +############# + +def progress_bar(count, total, asset_name='', previous_asset_name=''): + """ + This simple console progress bar + For display progress asset uploads + """ + + # This will prevent the characters from the previous asset name to be displayed, + # if the current asset name is shorter than the previous one. + text = f'{asset_name}'.ljust(len(previous_asset_name)) + + progress_line = '#' * int(round(50 * count / float(total))) + '-' * (50 - int(round(50 * count / float(total)))) + percent = round(100.0 * count / float(total), 1) + sys.stdout.write(f'[{progress_line}] {percent}% {text}\r') + sys.stdout.flush() + + +def set_token(value): + global token + token = f'Token {value}' + + +############ +# Database # +############ + +def get_assets_by_anthias_api(): + if click.confirm('Do you need authentication to access Anthias API?'): + login = click.prompt('Login') + password = click.prompt('Password', hide_input=True) + auth = HTTPBasicAuth(login, password) + else: + auth = None + response = requests.get(ASSETS_ANTHIAS_API, timeout=10, auth=auth) + + response.raise_for_status() + return response.json() + + +############ +# Requests # +############ + +@retry +def get_post_response(endpoint_url, **kwargs): + return requests.post(endpoint_url, **kwargs) + +def send_asset(asset): + endpoint_url = f'{BASE_API_SCREENLY_URL}/api/v4/assets' + asset_uri = asset['uri'] + post_kwargs = { + 'data': {'title': asset['name']}, + 'headers': { + 'Authorization': token, + 'Prefer': 'return=representation' + } + } + + try: + if asset['mimetype'] in ['image', 'video']: + if asset_uri.startswith('/data'): + asset_uri = os.path.join(HOME, 'screenly_assets', os.path.basename(asset_uri)) + + post_kwargs.update({ + 'files': { + 'file': open(asset_uri, 'rb') + } + }) + else: + post_kwargs['data'].update({'source_url': asset_uri}) + except FileNotFoundError as error: + click.secho(f'No such file or directory: {error.filename}', fg='red') + return False + + try: + response = get_post_response(endpoint_url, **post_kwargs) + response.raise_for_status() + except RequestException as error: + click.secho(f'Error: {error}', fg='red') + return False + + return True + + +def check_validate_token(api_key): + endpoint_url = f'{BASE_API_SCREENLY_URL}/api/v4/assets' + headers = { + 'Authorization': f'Token {api_key}' + } + response = requests.get(endpoint_url, headers=headers) + if response.status_code == 200: + return api_key + else: + return None + + +######## +# Main # +######## + +def start_migration(): + if click.confirm('Do you want to start assets migration?'): + assets_migration() + + +def assets_migration(): + try: + assets = get_assets_by_anthias_api() + except RequestException as error: + click.secho(f'Error: {error}', fg='red') + sys.exit(1) + + assets_length = len(assets) + failed_assets_count = 0 + previous_asset_name = '' + + click.echo('\n') + + for index, asset in enumerate(assets): + asset_name = str(asset['name']) + shortened_asset_name = shorten(asset_name, MAX_ASSET_NAME_LENGTH) + progress_bar( + index + 1, + assets_length, + asset_name=shortened_asset_name, + previous_asset_name=previous_asset_name + ) + previous_asset_name = shortened_asset_name + + status = send_asset(asset) + if not status: + failed_assets_count += 1 + click.secho(f'Failed to migrate asset: {asset_name}', fg='red') + + click.echo('\n') + + if failed_assets_count > 0: + click.secho(f'Migration completed with {failed_assets_count} failed assets', fg='red') + else: + click.secho('Migration completed successfully', fg='green') + + +@click.command() +@click.option( + '--method', + prompt=cleandoc( + """ + What do you want to use for migration? + 1. API token + 2. Exit + Your choice + """ + ), + type=click.Choice(['1', '2']) +) +def main(method): + try: + valid_token = None + + if method == '1': + api_key = click.prompt('Your API key') + valid_token = check_validate_token(api_key) + elif method == '2': + sys.exit(0) + + if valid_token: + set_token(valid_token) + click.secho('Successfull authentication', fg='green') + start_migration() + else: + click.secho('Failed authentication', fg='red') + except Exception: + traceback.print_exc() + + +if __name__ == '__main__': + click.secho(cleandoc(""" + d8888 888 888 + d88888 888 888 888 + d88P888 888 888 + d88P 888 88888b. 888888 88888b. 888 8888b. .d8888b + d88P 888 888 '88b 888 888 '88b 888 '88b 88K + d88P 888 888 888 888 888 888 888 .d888888 'Y8888b. + d8888888888 888 888 Y88b. 888 888 888 888 888 X88 + d88P 888 888 888 Y888 888 888 888 'Y888888 88888P' + """), fg='cyan') + + click.echo() + + main()