Scrape & Format Ciphersuites #15
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Scrape & Format Ciphersuites | |
| on: | |
| schedule: | |
| - cron: "0 0 * * 0" | |
| jobs: | |
| scrape-datasources: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| #- uses: actions/checkout@v2 | |
| - name: Setup Python 3.x environment | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: 3.9 | |
| - name: Clone cipherscrape github repo | |
| uses: GuillaumeFalourd/clone-github-repo-action@v2 | |
| with: | |
| owner: Spacecow99 | |
| repository: cipherscrape | |
| - name: Install cipherscrape and dependencies | |
| shell: bash | |
| run: | | |
| python -m pip install cipherscrape/ | |
| python -m cipherscrape --version | |
| - name: Scrape IANA reserved ciphersuite list | |
| shell: bash | |
| run: | | |
| python -m cipherscrape.iana > iana_ciphers.yaml | |
| - name: Scrape scanigma knowledge base | |
| shell: bash | |
| run: | | |
| python -m cipherscrape.scanigma > scanigma_ciphers.yaml | |
| - name: Scrape TestSSL.sh cipher-mapping.txt | |
| shell: bash | |
| run: | | |
| python -m cipherscrape.testssl > testssl_ciphers.yaml | |
| - name: Scrape openssl documentation and testssl cipher-mapping.txt | |
| shell: bash | |
| run: | | |
| python -m cipherscrape.openssl > openssl_ciphers.yaml | |
| - name: Scrape gnutls manual | |
| shell: bash | |
| run: | | |
| python -m cipherscrape.gnutls "manual" > gnutls_manual.yaml | |
| - name: Scrape gnutls ciphersuites.c source code | |
| shell: bash | |
| run: | | |
| python -m cipherscrape.gnutls "ciphersuites.c" > gnutls_source.yaml | |
| - name: Archive raw files as artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: raw-datasets | |
| path: "*.yaml" | |
| process-datasets: | |
| needs: scrape-datasources | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Setup Python 3.x environment | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: 3.9 | |
| - name: Clone complete_cs_instance github repo | |
| uses: GuillaumeFalourd/clone-github-repo-action@v2 | |
| with: | |
| owner: Spacecow99 | |
| repository: complete_cs_instance | |
| - name: Install complete_cs_instance and dependencies | |
| shell: bash | |
| run: | | |
| python -m pip install complete_cs_instance/ | |
| - name: Create directory structure | |
| shell: bash | |
| run: | | |
| mkdir -p ~/raw/ ~/processed/ | |
| - name: Download raw artifacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: raw-datasets | |
| path: ~/raw/ | |
| - name: Copy openssl_ciphers.yaml from ~/raw/ to ~/processed/ | |
| shell: bash | |
| run: | | |
| cp ~/raw/openssl_ciphers.yaml ~/processed/openssl_ciphers.yaml | |
| - name: Combine and deduplicate GNUTLS lists | |
| shell: bash | |
| run: | | |
| python deduplicate_lists.py ~/raw/gnutls_source.yaml ~/raw/gnutls_manual.yaml > ~/processed/gnutls_ciphers.yaml | |
| - name: Combine and deduplicate TestSSL & Scanigma lists | |
| shell: bash | |
| run: | | |
| python deduplicate_lists.py ~/raw/testssl_ciphers.yaml ~/raw/scanigma_ciphers.yaml > ~/processed/combined_3rd_party_ciphers.yaml | |
| - name: Combine and deduplicate IANA & combined 3rd party lists | |
| shell: bash | |
| run: | | |
| python deduplicate_lists.py ~/raw/iana_ciphers.yaml ~/processed/combined_3rd_party_ciphers.yaml > ~/processed/combined_iana_3rd_party_ciphers.yaml | |
| - name: Run combined_cs_instance on combined IANA & 3rd party list | |
| shell: bash | |
| run: | | |
| python -m complete_cs_instance ~/processed/combined_iana_3rd_party_ciphers.yaml > ~/processed/ciphersuites.yaml | |
| - name: Archive processed files as artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: processed-datasets | |
| path: "~/processed/*.yaml" | |
| upload-datasets: | |
| needs: process-datasets | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Download processed artifacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: processed-datasets | |
| path: ~/processed/ | |
| - name: Ensure openssl and gnutls folders exist | |
| shell: bash | |
| run: | | |
| mkdir -p fixtures/openssl fixtures/gnutls | |
| - name: Update repo's openssl_ciphers.yaml with artifact version | |
| shell: bash | |
| run: | | |
| cp -f ~/processed/openssl_ciphers.yaml fixtures/openssl/openssl_ciphers.yaml | |
| - name: Update repo's gnutls_ciphers.yaml with artifact version | |
| shell: bash | |
| run: | | |
| cp -f ~/processed/gnutls_ciphers.yaml fixtures/gnutls/gnutls_ciphers.yaml | |
| - name: Update repo's ciphersuites.yaml with artifact version | |
| shell: bash | |
| run: | | |
| cp -f ~/processed/ciphersuites.yaml fixtures/ciphersuites.yaml | |
| - name: Push changes to repo | |
| shell: bash | |
| run: | | |
| git config user.name Spacecow99 | |
| git config user.email [email protected] | |
| git add . | |
| if [[ -n "$(git status | grep modified)" ]]; then | |
| git commit -m "Updated fixtures from job @ $(date)" | |
| git push | |
| fi |