From e131694dbca37a17cc8253b855ff2f6eafc147f6 Mon Sep 17 00:00:00 2001 From: Gerolf Ziegenhain Date: Tue, 28 May 2024 17:47:37 +0200 Subject: [PATCH 1/6] workflow --- .github/workflows/docker.yml | 79 ++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 .github/workflows/docker.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..fb45810 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,79 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Create and publish a Docker image + +on: + push: + branches: + - 'master' + - 'main' + - 'dev' + + tags: + - 'v*' + - 'v*.*' + - 'v*.*.*' + - '*' + - '*.*' + - '*.*.*' + pull_request: + branches: + - 'main' + - 'dev' + + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # https://github.com/docker/setup-qemu-action + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + + - name: Log in to the Container registry + uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=sha + type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }} + + + - name: Build and push Docker image + uses: docker/build-push-action@v2 + with: + context: . + push: true + platforms: linux/amd64,linux/arm/v7 + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} From c85dec2e7b897b72c8633d38dda44eb6c175852d Mon Sep 17 00:00:00 2001 From: Gerolf Ziegenhain Date: Sat, 2 Nov 2024 14:08:03 +0100 Subject: [PATCH 2/6] use gtts and self contained docker file --- .gitignore | 14 +++++++++ Dockerfile | 7 ++++- README.md | 15 ++------- morse-code-ninja.sh | 3 -- requirements.txt | 2 ++ text2speech.py | 75 +++++++++++++++++++++++++-------------------- 6 files changed, 67 insertions(+), 49 deletions(-) delete mode 100755 morse-code-ninja.sh create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore index 52070a1..64ef27a 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,17 @@ venv/ # Don't check in rendered mp3 files *.mp3 +example-00001.txt +example-sentences.txt +example-structure.txt +sentence-15.txt +sentence-17.txt +sentence-20.txt +sentence-22.txt +sentence-25.txt +sentence-28.txt +sentence-30.txt +sentence-35.txt +sentence-40.txt +sentence-45.txt +sentence-50.txt diff --git a/Dockerfile b/Dockerfile index f2dcdb9..fc71efa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,12 @@ FROM python:3 RUN apt update RUN apt -y install lame ffmpeg ebook2cw -RUN pip install boto3 +RUN pip3 install --upgrade pip + +ADD requirements.txt . +RUN pip3 install -r requirements.txt RUN mkdir -p /opt/morse-code-ninja WORKDIR /opt/morse-code-ninja + +ADD . . diff --git a/README.md b/README.md index e4fafe6..b50d77a 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,9 @@ you may define AWS_KEY_ID and AWS_SECRET_ACCESS_KEY as environmental variables. 4. Run this command to make sure that you don't accidentally check in your key! `git update-index --assume-unchanged aws.properties` +##### GZ +docker run --rm -it -v $PWD/mp3:/opt/morse-code-ninja/mp3 mp3 perl render.pl -i example.txt -o mp3 + # Usage #### EXAMPLE: @@ -82,15 +85,3 @@ multiple copies are executing at the same time using the same output directory. Be aware that the script can create a huge number of temporary files, which is proportional to the input file. Some types of filesystems will deal with this better than others. This set of scripts works on Linux and macOS. - -# Docker Usage -This usage limits the required software to: -- [Docker](https://www.docker.com/get-started/) -- [AWS Cloud Setup](#cloud-setup) - -To run -``` -./morse-code-ninja.sh -i example.txt -``` - -The script `morse-code-ninja.sh` wraps the docker compose command passing all arguments to the dockerized `render.pl` diff --git a/morse-code-ninja.sh b/morse-code-ninja.sh deleted file mode 100755 index 2ed92f1..0000000 --- a/morse-code-ninja.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -docker compose run --rm app perl render.pl "$@" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..fac5658 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +gtts +boto3 \ No newline at end of file diff --git a/text2speech.py b/text2speech.py index 85f082a..2a14dbc 100755 --- a/text2speech.py +++ b/text2speech.py @@ -8,6 +8,8 @@ from os import environ import shutil import subprocess +from gtts import gTTS + sentence_filename = sys.argv[1] engine_type = sys.argv[2].lower() # needs to be: standard | neural @@ -27,28 +29,28 @@ separator = "=" aws_properties = {} -if "AWS_ACCESS_KEY_ID" in environ and "AWS_SECRET_ACCESS_KEY" in environ: - aws_properties['aws_access_key_id'] = environ['AWS_ACCESS_KEY_ID'] - aws_properties['aws_secret_access_key'] = environ['AWS_SECRET_ACCESS_KEY'] -else: - try: - with open('aws.properties') as property_file: - - for line in property_file: - if separator in line: - name, value = line.split(separator, 1) - aws_properties[name.strip()] = value.strip() - except IOError as e: - print(f"I/O error reading aws.properties: {e.errno}, {e.strerror}") - - print("text2speech script does not have AWS credentials.") - sys.exit(ioError) - -if aws_properties['aws_access_key_id'] == 'CHANGE_ME' or aws_properties['aws_secret_access_key'] == 'CHANGE ME': - print("**********************************************************************************") - print("text2speech script does NOT have AWS credentials. Set properties in aws.properties") - print("**********************************************************************************") - sys.exit(ioError) +# if "AWS_ACCESS_KEY_ID" in environ and "AWS_SECRET_ACCESS_KEY" in environ: +# aws_properties['aws_access_key_id'] = environ['AWS_ACCESS_KEY_ID'] +# aws_properties['aws_secret_access_key'] = environ['AWS_SECRET_ACCESS_KEY'] +# else: +# try: +# with open('aws.properties') as property_file: + +# for line in property_file: +# if separator in line: +# name, value = line.split(separator, 1) +# aws_properties[name.strip()] = value.strip() +# except IOError as e: +# print(f"I/O error reading aws.properties: {e.errno}, {e.strerror}") + +# print("text2speech script does not have AWS credentials.") +# sys.exit(ioError) + +# if aws_properties['aws_access_key_id'] == 'CHANGE_ME' or aws_properties['aws_secret_access_key'] == 'CHANGE ME': +# print("**********************************************************************************") +# print("text2speech script does NOT have AWS credentials. Set properties in aws.properties") +# print("**********************************************************************************") +# sys.exit(ioError) sha256_hash = hashlib.sha256() @@ -62,18 +64,25 @@ def render(cache_filename, voice_id, text_type, text): if not os.path.exists(cache_filename): - polly_client = boto3.Session(aws_access_key_id=aws_properties['aws_access_key_id'], - aws_secret_access_key=aws_properties['aws_secret_access_key'], - region_name='us-east-1').client('polly') - if text_type is None: - response = polly_client.synthesize_speech(Engine=engine_type, VoiceId=voice_id, OutputFormat='mp3', Text=text) - else: - response = polly_client.synthesize_speech(Engine=engine_type, VoiceId=voice_id, OutputFormat='mp3', - TextType=text_type, Text=text) + # polly_client = boto3.Session(aws_access_key_id=aws_properties['aws_access_key_id'], + # aws_secret_access_key=aws_properties['aws_secret_access_key'], + # region_name='us-east-1').client('polly') + # if text_type is None: + # response = polly_client.synthesize_speech(Engine=engine_type, VoiceId=voice_id, OutputFormat='mp3', Text=text) + # else: + # response = polly_client.synthesize_speech(Engine=engine_type, VoiceId=voice_id, OutputFormat='mp3', + # TextType=text_type, Text=text) + + # file = open(temp_filename, 'wb') + # file.write(response['AudioStream'].read()) + # file.close() + + print ("Create the mp3 file") + language = 'en' + myobj = gTTS(text=text, lang=language, slow=False) + myobj.save(temp_filename) + - file = open(temp_filename, 'wb') - file.write(response['AudioStream'].read()) - file.close() subprocess.run(['lame', '--resample', '44.1', '-a', '-b', '256', temp_filename, From a18d7ea65b0ceae53bf15a3ee606826f845697f8 Mon Sep 17 00:00:00 2001 From: Gerolf Ziegenhain Date: Sat, 2 Nov 2024 14:27:52 +0100 Subject: [PATCH 3/6] wip - file structure output - dnf --- .gitignore | 1 + Dockerfile | 1 + README.md | 11 ++++++----- aws.properties | 2 -- docker-compose.yml | 10 ---------- entrypoint.sh | 3 +++ 6 files changed, 11 insertions(+), 17 deletions(-) delete mode 100644 aws.properties delete mode 100644 docker-compose.yml create mode 100755 entrypoint.sh diff --git a/.gitignore b/.gitignore index 64ef27a..8d9a9ba 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ venv/ *.log # Don't check in rendered mp3 files +mp3/ *.mp3 example-00001.txt example-sentences.txt diff --git a/Dockerfile b/Dockerfile index fc71efa..03cba3a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,5 +9,6 @@ RUN pip3 install -r requirements.txt RUN mkdir -p /opt/morse-code-ninja WORKDIR /opt/morse-code-ninja +RUN mkdir mp3 ADD . . diff --git a/README.md b/README.md index b50d77a..16d753d 100644 --- a/README.md +++ b/README.md @@ -26,11 +26,14 @@ you may define AWS_KEY_ID and AWS_SECRET_ACCESS_KEY as environmental variables. 4. Run this command to make sure that you don't accidentally check in your key! `git update-index --assume-unchanged aws.properties` -##### GZ -docker run --rm -it -v $PWD/mp3:/opt/morse-code-ninja/mp3 mp3 perl render.pl -i example.txt -o mp3 - # Usage +##### Run in docker + docker build . -t mp3 + docker run --rm -it -v $PWD/mp3:/opt/morse-code-ninja/mp3 mp3 perl render.pl -i example.txt -o mp3 + docker run --rm -it -v $PWD/mp3:/opt/morse-code-ninja/mp3 mp3 ./entrypoint.sh -i practice-sets/instant-qso-element-courses/Course\ 01\ -\ Lesson\ 022\ -\ RST\ -\ Introduce\ New\ Element.txt + + #### EXAMPLE: perl render.pl -i example.txt @@ -43,8 +46,6 @@ docker run --rm -it -v $PWD/mp3:/opt/morse-code-ninja/mp3 mp3 perl render.pl -i [--norepeat] [--nospoken] [--nocourtesytone] [-e NEURAL | STANDARD] [--sm] [--ss] [--sv] [-x] [--lang ENGLISH | SWEDISH] -Uses AWS Polly and requires valid credentials in the aws.properties file.

- #### OPTIONS: ##### Required: diff --git a/aws.properties b/aws.properties deleted file mode 100644 index 55af0f0..0000000 --- a/aws.properties +++ /dev/null @@ -1,2 +0,0 @@ -aws_access_key_id=CHANGE_ME -aws_secret_access_key=CHANGE_ME diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 680383b..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,10 +0,0 @@ -version: '3' -services: - app: - container_name: morse-code-ninja - build: . - volumes: - - .:/opt/morse-code-ninja/ - environment: - - AWS_KEY_ID=${AWS_ACCESS_KEY_ID} - - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100755 index 0000000..460c8b1 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,3 @@ +#!/bin/bash +perl render.pl "$@" +mv *.mp3 ./mp3 \ No newline at end of file From 769d0b1785518f4c3418075ae5fb3937fee7ace3 Mon Sep 17 00:00:00 2001 From: Gerolf Ziegenhain Date: Sat, 2 Nov 2024 14:45:35 +0100 Subject: [PATCH 4/6] macos setup --- .gitignore | 1 + README.md | 29 +++++++---------------------- text2speech.py | 2 +- upload2s3.py | 2 +- 4 files changed, 10 insertions(+), 24 deletions(-) diff --git a/.gitignore b/.gitignore index 8d9a9ba..9c6d1d6 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ aws.properties* # python venv venv/ +.venv/ *.log diff --git a/README.md b/README.md index 16d753d..14a18d7 100644 --- a/README.md +++ b/README.md @@ -4,34 +4,19 @@ The software used to generate Morse Code Ninja practice sets as found on [Kurt Zoglmann's YouTube Channel](https://www.youtube.com/channel/UCXrTMfMEhkC9rVyQNU5aZlA). # Required Software -These must be installed and available in your Shell's PATH. -* [ebook2cw](https://fkurz.net/ham/ebook2cw.html) -* [ffmpeg](https://ffmpeg.org) -* [lame](https://lame.sourceforge.io/) -* [Perl 5](https://www.perl.org) -* [Python 3](https://www.python.org) -* [Boto3](https://aws.amazon.com/sdk-for-python/) -# Cloud Setup -1. Set up an AWS Account. Feel free to follow these -[instructions](https://aws.amazon.com/premiumsupport/knowledge-center/create-and-activate-aws-account/). - -2. Create an [IAM user](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_users_create.html). -For ease, I recommend using the "console" for this. During creation, give -the IAM user "Programmatic access." When prompted save the **key ID** and **secret access key**. -During the creation, attach the **AmazonPollyFullAccess** policy to the user. - -3. Edit the aws.properties file. Set the **key ID** and **secret access key**. As an alternative, -you may define AWS_KEY_ID and AWS_SECRET_ACCESS_KEY as environmental variables. - -4. Run this command to make sure that you don't accidentally check in your key! `git update-index --assume-unchanged aws.properties` +# Setup macOS +python3 -m venv .venv +source .venv/bin/activate +pip3 install -r requirements.txt +brew install ebook2cw lame ffmpeg +perl render.pl -i practice-sets/instant-qso-element-courses/Course\ 01\ -\ Lesson\ 022\ -\ RST\ -\ Introduce\ New\ Element.txt # Usage -##### Run in docker +##### Run in docker (WIP) docker build . -t mp3 docker run --rm -it -v $PWD/mp3:/opt/morse-code-ninja/mp3 mp3 perl render.pl -i example.txt -o mp3 - docker run --rm -it -v $PWD/mp3:/opt/morse-code-ninja/mp3 mp3 ./entrypoint.sh -i practice-sets/instant-qso-element-courses/Course\ 01\ -\ Lesson\ 022\ -\ RST\ -\ Introduce\ New\ Element.txt #### EXAMPLE: diff --git a/text2speech.py b/text2speech.py index 2a14dbc..19362be 100755 --- a/text2speech.py +++ b/text2speech.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import boto3 import sys diff --git a/upload2s3.py b/upload2s3.py index d73d284..6c0de24 100644 --- a/upload2s3.py +++ b/upload2s3.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import boto3 import os From ed419ea68ee895e3549d795707f59111f6d12ed1 Mon Sep 17 00:00:00 2001 From: Gerolf Ziegenhain Date: Sat, 2 Nov 2024 20:37:14 +0100 Subject: [PATCH 5/6] vs code --- Morse-Code-Ninja.code-workspace | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 Morse-Code-Ninja.code-workspace diff --git a/Morse-Code-Ninja.code-workspace b/Morse-Code-Ninja.code-workspace new file mode 100644 index 0000000..876a149 --- /dev/null +++ b/Morse-Code-Ninja.code-workspace @@ -0,0 +1,8 @@ +{ + "folders": [ + { + "path": "." + } + ], + "settings": {} +} \ No newline at end of file From 2b09cfd8880afec82cfdf421cff1ecb5d619e20f Mon Sep 17 00:00:00 2001 From: Gerolf Ziegenhain Date: Sat, 2 Nov 2024 20:51:40 +0100 Subject: [PATCH 6/6] wip with single file --- Dockerfile | 4 ++++ README.md | 2 +- entrypoint.sh | 4 +++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 03cba3a..0b9c513 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,5 +10,9 @@ RUN pip3 install -r requirements.txt RUN mkdir -p /opt/morse-code-ninja WORKDIR /opt/morse-code-ninja RUN mkdir mp3 +RUN mkdir /input +RUN mkdir /output ADD . . + +#ENTRYPOINT ["./entrypoint.sh", "-i", "input.txt"] diff --git a/README.md b/README.md index 14a18d7..4becc2b 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ perl render.pl -i practice-sets/instant-qso-element-courses/Course\ 01\ -\ Lesso # Usage ##### Run in docker (WIP) docker build . -t mp3 - docker run --rm -it -v $PWD/mp3:/opt/morse-code-ninja/mp3 mp3 perl render.pl -i example.txt -o mp3 + docker run --rm -it -v $PWD/practice-sets/instant-qso-element-courses/Course\ 01\ -\ Lesson\ 022\ -\ RST\ -\ Introduce\ New\ Element.txt:/input/input.txt:ro -v $PWD/mp3:/output mp3 ./entrypoint.sh -i input.txt #### EXAMPLE: diff --git a/entrypoint.sh b/entrypoint.sh index 460c8b1..c3d2192 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,3 +1,5 @@ #!/bin/bash +cp /input/* . perl render.pl "$@" -mv *.mp3 ./mp3 \ No newline at end of file +mv *.mp3 /output +