diff --git a/gh-starred-to-opml.sh b/gh-starred-to-opml.sh new file mode 100644 index 0000000..9dc036f --- /dev/null +++ b/gh-starred-to-opml.sh @@ -0,0 +1,191 @@ +#!/usr/bin/env bash + +shopt -s extglob + +export LC_ALL="C.UTF-8" +export TZ=:/etc/localtime + +Help() +{ + echo "Generate an OPML 2.0 file to follow releases of starred repositories on Github" + echo "Default to all starred repos of the user, or a specific list with [-l list]." + echo + echo "Syntax: ./gh-starred-to-opml.sh [-h] -u user [-l listname] [-d date] [-o filename] [-n filename]" + echo "options:" + echo "-h Print this Help" + echo "-u string required: Github username" + echo "-l string optional: Github Stars List (url shortname)" + echo "-d string optional: ISO8601 date to filter out starred before (YYYY-mm-ddTHH:MM:SSZ)" + echo "-o string.opml optional: destination opml filename" + echo "-n string.opml Use a previous file to generate updates," + echo " it does not require any other option," + echo " but it uses starred_at dates, not starred in a list." + echo +} + +unset -v _USERNAME _LISTNAME _DATEFILTER _FILENAME _OLDFILENAME _DATA + +while getopts "hu:l:d:o:n:" option; do + case $option in + h) Help; exit;; + u) _USERNAME="$OPTARG";; + l) _LISTNAME="$OPTARG";; + d) _DATEFILTER="$OPTARG";; + o) _FILENAME="$OPTARG";; + n) _OLDFILENAME="$OPTARG" + unset -v _USERNAME _LISTNAME _DATEFILTER _FILENAME _DATA;; + \?) echo -e "Unknown option: -$OPTARG \n" >&2; Help; exit 1;; + : ) echo -e "Missing argument for -$OPTARG \n" >&2; Help; exit 1;; + * ) echo -e "Unimplemented option: -$option \n" >&2; Help; exit 1;; + esac +done + +if [[ -n $_OLDFILENAME ]]; then + # parsing the source OPML file to generate values for the update + if [[ ! -f "${_OLDFILENAME}" ]]; then + echo 'Missing source file "${_OLDFILENAME}"' >&2 + echo + Help + exit 1 + fi + _FILENAME=${_OLDFILENAME/%*(_+([0-9]).opml)*(.opml)/_$(date '+%y%m%d%H%M').opml} + _USERNAME=$(grep "ownerName" "${_OLDFILENAME}" | sed -nr 's,.+Name>(.+)<\/ownerName.+,\1,p') + _LISTNAME=$(grep "outline" "${_OLDFILENAME}" | sed -nr 's,.+Github - (.+) - .+,\1,p') + _DATEFILTER=$(LC_ALL="C.UTF-8" date -u -d "$(grep "dateModified" "${_OLDFILENAME}" | sed -nr 's,.+dateModified>(.+)<\/dateModified.+,\1,p')" '+%Y-%m-%dT%H:%M:%S%Z') +fi + +if [[ -z $_USERNAME ]]; then + echo 'Missing Github username' >&2 + echo + Help + exit 1 +fi + +if [[ -z $_FILENAME ]]; then + if [[ -z $_LISTNAME ]]; then + _FILENAME="gh_starred_${_USERNAME}.opml" + else + _FILENAME="gh_starred_${_USERNAME}_${_LISTNAME}.opml" + fi +fi + + +_CMD_ARRAY=( curl jq pup ) +for cmd in "${_CMD_ARRAY[@]}"; do + if [[ -z $(command -v $cmd) ]]; then + echo 'Requirements: $cmd could not be found' >&2 + echo ' - sudo apt install curl jq' >&2 + echo ' - go install github.com/ericchiang/pup@latest' >&2 + exit 1 + fi +done + +# parsing Github API for all user's starred repositories +_PAGE=1 + +echo "- Parsing Github API for ${_USERNAME}: page $_PAGE" +_DATA=$(curl -H "Accept: application/vnd.github.v3.star+json" -sSL "api.github.com/users/${_USERNAME}/starred?page=${_PAGE}&per_page=100" | jq '.[] | {id: .repo.id, name: .repo.full_name, desc: .repo.description, date: .starred_at}') +_PAGE=$((_PAGE+1)) + +while [ $_PAGE -ge 2 ]; do + unset _NEW_DATA + echo "- Parsing Github API for ${_USERNAME}: page $_PAGE" + _NEW_DATA=$(curl -H "Accept: application/vnd.github.v3.star+json" -sSL "api.github.com/users/${_USERNAME}/starred?page=${_PAGE}&per_page=100" | jq '.[] | {id: .repo.id, name: .repo.full_name, desc: .repo.description, date: .starred_at}') + if [ "$_NEW_DATA" ]; then + _DATA=$(echo -e "${_DATA}\n${_NEW_DATA}") + _PAGE=$((_PAGE+1)) + else + echo "- Page ${_PAGE} was the last one" + _PAGE=1 + fi +done + +echo "- Sorting results by name" +_DATA=$(echo "${_DATA}" | jq -s -c 'sort_by(.name) | .[]') + +if [[ -n $_DATEFILTER ]]; then + echo "- Filtering out repositories starred before ${_DATEFILTER}" + _DATA=$(echo "${_DATA}" | jq -c --arg date "${_DATEFILTER}" 'select( .date > $date )') +fi + +# parsing Github list pages (not available in the API) +if [[ -n $_LISTNAME ]]; then + _PAGE=1 + + echo "- Parsing Github \"${_LISTNAME}\" List for ${_USERNAME}: page $_PAGE" + _LIST=$(curl -sS "https://github.com/stars/${_USERNAME}/lists/${_LISTNAME}?page=1") + _NEXT=$(echo "${_LIST}" | pup '#user-list-repositories div div a[class="next_page"] text{}') + _LIST=$(echo "${_LIST}" | pup '#user-list-repositories div div h3 a attr{href}') + if [[ -n $_NEXT ]]; then + _PAGE=$((_PAGE+1)) + fi + + while [ $_PAGE -ge 2 ]; do + unset _NEW_LIST _NEXT + echo "- Parsing Github \"${_LISTNAME}\" List for ${_USERNAME}: page $_PAGE" + _NEW_LIST=$(curl -sS "https://github.com/stars/${_USERNAME}/lists/${_LISTNAME}?page=${_PAGE}") + _NEXT=$(echo "${_NEW_LIST}" | pup '#user-list-repositories div div a[class="next_page"] text{}') + _NEW_LIST=$(echo "${_NEW_LIST}" | pup '#user-list-repositories div div h3 a attr{href}') + _LIST=$(echo -e "${_LIST}\n${_NEW_LIST}") + if [[ -n $_NEXT ]]; then + _PAGE=$((_PAGE+1)) + else + echo "- Page ${_PAGE} was the last one" + _PAGE=1 + fi + done + + _LIST=$(echo "${_LIST}" | sed -r 's,^\/,,g') +fi + +# filtering down repositories by the List +if [[ -n $_LIST ]]; then + echo "- Filtering all starred repositories by the ones in the list." + _DATA=$(echo "${_DATA}" | jq --argjson names "$(echo "${_LIST}" | jq -R . | jq -s .)" 'select( .name as $name | $names | index($name) )') +fi + +if [[ "$(echo "${_DATA}" | jq '.name' | wc -l)" == "0" ]]; then + echo 'No entries found. Aborting.' >&2 + echo + exit 1 +fi + +# opml file generation +echo "- Generating OPML file Header" +_FIRSTDATE=$(LC_ALL="C.UTF-8" TZ=GMT date -d $(echo "${_DATA}" | jq -sr 'sort_by(.date) | reverse[-1].date') '+%a, %d %b %Y %H:%M:%S %Z') +_LASTDATE=$(LC_ALL="C.UTF-8" TZ=GMT date -d $(echo "${_DATA}" | jq -sr 'sort_by(.date)[-1].date') '+%a, %d %b %Y %H:%M:%S %Z') +if [[ -z $_LISTNAME ]]; then + _CATEGORY="Github - ${_USERNAME}" +else + _CATEGORY="Github - ${_LISTNAME} - ${_USERNAME}" +fi +cat <<- EOF > "${_FILENAME}" + + + + ${_FILENAME} + ${_FIRSTDATE} + ${_LASTDATE} + ${_USERNAME} + + + +EOF + +echo "- Generating OPML file Feeds" +for id in $(echo "$_DATA" | jq .id); do + _REPO_NAME=$(echo "$_DATA" | jq -r --argjson v "$id" ' . | select(.id==$v).name') + _REPO_DESC=$(echo "$_DATA" | jq -r --argjson v "$id" ' . | select(.id==$v).desc' | sed -e 's~\&~\&~g' -e 's~<~\<~g' -e 's~>~\>~g' -e 's~\"~\"~g' -e "s~'~\'~g") + cat <<- EOF >> "${_FILENAME}" + +EOF +done + +echo "- Generating OPML file Footer" +cat <<- EOF >> "${_FILENAME}" + + + +EOF + +echo "- Done: Generated $(echo "${_DATA}" | jq '.name' | wc -l) entries for \"${_CATEGORY}\" in ${_FILENAME}"