#!/usr/bin/env bash shopt -s extglob export LC_ALL="C.UTF-8" export TZ=:/etc/localtime Help() { echo "Generate an OPML 2.0 file to follow releases of starred repositories on Github" echo "Default to all starred repos of the user, or a specific list with [-l list]." echo "Uses the Github API to check lists and if the starred repos contain Release (preferred) or Tag entries." echo echo "Syntax: ./gh-starred-to-opml.sh [-h] -u user [-l listname] [-d date] [-o filename] [-n filename] -t token" echo "options:" echo "-h Print this Help" echo "-u string required: Github username" echo "-l string optional: Github Stars List (url shortname)" echo "-d string optional: ISO8601 date to filter out starred before (YYYY-mm-ddTHH:MM:SSZ)" echo "-o string.opml optional: destination opml filename" echo "-n string.opml Use a previous file to generate updates," echo " it does not require any other option," echo " but it uses starred_at dates, not starred in a list." echo "-t string required: Github API token to avoid rate limits." echo } unset -v _USERNAME _LISTNAME _DATEFILTER _FILENAME _OLDFILENAME _DATA while getopts "hu:l:d:o:n:t:" option; do case $option in h) Help; exit;; u) _USERNAME="$OPTARG";; l) _LISTNAME="$OPTARG";; d) _DATEFILTER="$OPTARG";; o) _FILENAME="$OPTARG";; n) _OLDFILENAME="$OPTARG" unset -v _USERNAME _LISTNAME _DATEFILTER _FILENAME _DATA;; t) _GITHUBTOKEN="$OPTARG";; \?) echo -e "Unknown option: -$OPTARG \n" >&2; Help; exit 1;; : ) echo -e "Missing argument for -$OPTARG \n" >&2; Help; exit 1;; * ) echo -e "Unimplemented option: -$option \n" >&2; Help; exit 1;; esac done if [[ -n $_OLDFILENAME ]]; then # parsing the source OPML file to generate values for the update if [[ ! -f "${_OLDFILENAME}" ]]; then echo "Missing source file ${_OLDFILENAME}" >&2 echo Help exit 1 fi _FILENAME=${_OLDFILENAME/%*(_+([0-9]).opml)*(.opml)/_$(date '+%y%m%d%H%M').opml} _USERNAME=$(grep "ownerName" "${_OLDFILENAME}" | sed -nr 's,.+Name>(.+)<\/ownerName.+,\1,p') _LISTNAME=$(grep "outline" "${_OLDFILENAME}" | sed -nr 's,.+Github - (.+) - .+,\1,p') _DATEFILTER=$(LC_ALL="C.UTF-8" date -u -d "$(grep "dateModified" "${_OLDFILENAME}" | sed -nr 's,.+dateModified>(.+)<\/dateModified.+,\1,p')" '+%Y-%m-%dT%H:%M:%S%Z') fi if [[ -z $_USERNAME ]]; then echo 'Missing Github username' >&2 echo Help exit 1 fi if [[ -z $_GITHUBTOKEN ]]; then echo 'Missing Github API token' >&2 echo Help exit 1 fi if [[ -z $_FILENAME ]]; then if [[ -z $_LISTNAME ]]; then _FILENAME="gh_starred_${_USERNAME}.opml" else _FILENAME="gh_starred_${_USERNAME}_${_LISTNAME}.opml" fi fi _CMD_ARRAY=( curl jq pup ) for cmd in "${_CMD_ARRAY[@]}"; do if [[ -z $(command -v "$cmd") ]]; then echo "Requirements: $cmd could not be found" >&2 echo ' - sudo apt install curl jq' >&2 echo ' - go install github.com/ericchiang/pup@latest' >&2 exit 1 fi done trap "echo '⯅ Github API rate limit exceeded or Token issue.'; exit 1" TERM export MAIN_PID=$$ # request to Github API with rate limit trap ghapirequest() { local _GITHUB _GITHUB=$(curl \ --max-time 120 \ -H "Authorization: token ${_GITHUBTOKEN}" \ -H "Accept: application/vnd.github.v3.star+json" \ -sSL "https://api.github.com/$1") if (echo "${_GITHUB}" | grep -q 'API rate limit exceeded'); then kill -s TERM $MAIN_PID else echo "${_GITHUB}" fi } # request to Github lists ghrequest() { curl --max-time 120 -sSL "https://github.com/$1" } # parsing Github API for all user's starred repositories _PAGE=1 echo "- Parsing Github API for ${_USERNAME}: page $_PAGE" _DATA=$(ghapirequest "users/${_USERNAME}/starred?page=${_PAGE}&per_page=100" | jq '.[] | {id: .repo.id, name: .repo.full_name, desc: .repo.description, date: .starred_at}') _PAGE=$((_PAGE+1)) while [ $_PAGE -ge 2 ]; do unset _NEW_DATA echo "- Parsing Github API for ${_USERNAME}: page $_PAGE" _NEW_DATA=$(ghapirequest "users/${_USERNAME}/starred?page=${_PAGE}&per_page=100" | jq '.[] | {id: .repo.id, name: .repo.full_name, desc: .repo.description, date: .starred_at}') if [ "$_NEW_DATA" ]; then _DATA=$(echo -e "${_DATA}\n${_NEW_DATA}") _PAGE=$((_PAGE+1)) else echo "- Page ${_PAGE} was the last one" _PAGE=1 fi done echo "- Sorting results by name" _DATA=$(echo "${_DATA}" | jq -s -c 'sort_by(.name) | .[]') if [[ -n $_DATEFILTER ]]; then echo "- Filtering out repositories starred before ${_DATEFILTER}" _DATA=$(echo "${_DATA}" | jq -c --arg date "${_DATEFILTER}" 'select( .date > $date )') fi # parsing Github list pages (not available in the API) if [[ -n $_LISTNAME ]]; then _PAGE=1 echo "- Parsing Github \"${_LISTNAME}\" List for ${_USERNAME}: page $_PAGE" _LIST=$(ghrequest "stars/${_USERNAME}/lists/${_LISTNAME}?page=1") _NEXT=$(echo "${_LIST}" | pup '#user-list-repositories div div a[class="next_page"] text{}') _LIST=$(echo "${_LIST}" | pup '#user-list-repositories div div h3 a attr{href}') if [[ -n $_NEXT ]]; then _PAGE=$((_PAGE+1)) fi while [ $_PAGE -ge 2 ]; do unset _NEW_LIST _NEXT echo "- Parsing Github \"${_LISTNAME}\" List for ${_USERNAME}: page $_PAGE" _NEW_LIST=$(ghrequest "stars/${_USERNAME}/lists/${_LISTNAME}?page=${_PAGE}") _NEXT=$(echo "${_NEW_LIST}" | pup '#user-list-repositories div div a[class="next_page"] text{}') _NEW_LIST=$(echo "${_NEW_LIST}" | pup '#user-list-repositories div div h3 a attr{href}') _LIST=$(echo -e "${_LIST}\n${_NEW_LIST}") if [[ -n $_NEXT ]]; then _PAGE=$((_PAGE+1)) else echo "- Page ${_PAGE} was the last one" _PAGE=1 fi done _LIST=$(echo "${_LIST}" | sed -r 's,^\/,,g') fi # filtering down repositories by the List if [[ -n $_LIST ]]; then echo "- Filtering all starred repositories by the ones in the list." _DATA=$(echo "${_DATA}" | jq --argjson names "$(echo "${_LIST}" | jq -R . | jq -s .)" 'select( .name as $name | $names | index($name) )') fi if [[ "$(echo "${_DATA}" | jq '.name' | wc -l)" == "0" ]]; then echo 'No entries found. Aborting.' >&2 echo exit 1 fi # opml file generation echo "- Generating OPML file Header" _FIRSTDATE=$(LC_ALL="C.UTF-8" TZ=GMT date -d "$(echo "${_DATA}" | jq -sr 'sort_by(.date) | reverse[-1].date')" '+%a, %d %b %Y %H:%M:%S %Z') _LASTDATE=$(LC_ALL="C.UTF-8" TZ=GMT date -d "$(echo "${_DATA}" | jq -sr 'sort_by(.date)[-1].date')" '+%a, %d %b %Y %H:%M:%S %Z') if [[ -z $_LISTNAME ]]; then _CATEGORY="Github - ${_USERNAME}" else _CATEGORY="Github - ${_LISTNAME} - ${_USERNAME}" fi cat <<- EOF > "${_FILENAME}" ${_FILENAME} ${_FIRSTDATE} ${_LASTDATE} ${_USERNAME} EOF echo "- Generating OPML file Feeds" for id in $(echo "$_DATA" | jq .id); do _REPO_NAME=$(echo "$_DATA" | jq -r --argjson v "$id" ' . | select(.id==$v).name') _REPO_DESC=$(echo "$_DATA" | jq -r --argjson v "$id" ' . | select(.id==$v).desc' | sed -e 's~\&~\&~g' -e 's~<~\<~g' -e 's~>~\>~g' -e 's~\"~\"~g' -e "s~'~\'~g") _RELEASES=$(ghapirequest "repos/${_REPO_NAME}/releases" | jq '.[]') _TYPE="releases" if [[ -z "$_RELEASE" ]]; then _TAGS=$(ghapirequest "repos/${_REPO_NAME}/tags" | jq '.[]') if [[ -n "$_TAGS" ]]; then _TYPE="tags" fi fi cat <<- EOF >> "${_FILENAME}" EOF done echo "- Generating OPML file Footer" cat <<- EOF >> "${_FILENAME}" EOF echo "- Done: Generated $(echo "${_DATA}" | jq '.name' | wc -l) entries for \"${_CATEGORY}\" in ${_FILENAME}"