new: require Github API token, check for rate limiting

gh-starred-to-opml.sh

@@ -9,8 +9,9 @@ Help()
 {
   echo "Generate an OPML 2.0 file to follow releases of starred repositories on Github"
   echo "Default to all starred repos of the user, or a specific list with [-l list]."
+  echo "Uses the Github API to check lists and if the starred repos contain Release (preferred) or Tag entries."
   echo
-  echo "Syntax: ./gh-starred-to-opml.sh [-h] -u user [-l listname] [-d date] [-o filename] [-n filename]"
+  echo "Syntax: ./gh-starred-to-opml.sh [-h] -u user [-l listname] [-d date] [-o filename] [-n filename] -t token"
   echo "options:"
   echo "-h              Print this Help"
   echo "-u string       required: Github username"
@@ -20,12 +21,13 @@ Help()
   echo "-n string.opml  Use a previous file to generate updates,"
   echo "                it does not require any other option,"
   echo "                but it uses starred_at dates, not starred in a list."
+  echo "-t string       required: Github API token to avoid rate limits."
   echo
 }
 
 unset -v _USERNAME _LISTNAME _DATEFILTER _FILENAME _OLDFILENAME _DATA
 
-while getopts "hu:l:d:o:n:" option; do
+while getopts "hu:l:d:o:n:t:" option; do
   case $option in
      h) Help; exit;;
      u) _USERNAME="$OPTARG";;
@@ -34,6 +36,7 @@ while getopts "hu:l:d:o:n:" option; do
      o) _FILENAME="$OPTARG";;
      n) _OLDFILENAME="$OPTARG"
         unset -v _USERNAME _LISTNAME _DATEFILTER _FILENAME _DATA;;
+     t) _GITHUBTOKEN="$OPTARG";;
     \?) echo -e "Unknown option: -$OPTARG \n" >&2; Help; exit 1;;
     : ) echo -e "Missing argument for -$OPTARG \n" >&2; Help; exit 1;;
     * ) echo -e "Unimplemented option: -$option \n" >&2; Help; exit 1;;
@@ -43,7 +46,7 @@ done
 if [[ -n $_OLDFILENAME ]]; then
   # parsing the source OPML file to generate values for the update
   if [[ ! -f "${_OLDFILENAME}" ]]; then
-    echo 'Missing source file "${_OLDFILENAME}"' >&2
+    echo "Missing source file ${_OLDFILENAME}" >&2
     echo
     Help
     exit 1
@@ -61,36 +64,66 @@ if [[ -z $_USERNAME ]]; then
   exit 1
 fi
 
+if [[ -z $_GITHUBTOKEN ]]; then
+  echo 'Missing Github API token' >&2
+  echo
+  Help
+  exit 1
+fi
+
 if [[ -z $_FILENAME ]]; then
   if [[ -z $_LISTNAME ]]; then
-    _FILENAME="gh_starred_${_USERNAME}_$(date '+%y%m%d%H%M').opml"
+    _FILENAME="gh_starred_${_USERNAME}.opml"
   else
-    _FILENAME="gh_starred_${_USERNAME}_${_LISTNAME}_$(date '+%y%m%d%H%M').opml"
+    _FILENAME="gh_starred_${_USERNAME}_${_LISTNAME}.opml"
   fi
 fi
 
 
 _CMD_ARRAY=( curl jq pup )
 for cmd in "${_CMD_ARRAY[@]}"; do
-  if [[ -z $(command -v $cmd) ]]; then
+  if [[ -z $(command -v "$cmd") ]]; then
-    echo 'Requirements: $cmd could not be found' >&2
+    echo "Requirements: $cmd could not be found" >&2
     echo '  - sudo apt install curl jq' >&2
     echo '  - go install github.com/ericchiang/pup@latest' >&2
     exit 1
   fi
 done
 
+trap "echo '⯅ Github API rate limit exceeded or Token issue.'; exit 1" TERM
+export MAIN_PID=$$
+
+# request to Github API with rate limit trap
+ghapirequest() {
+  local _GITHUB
+  _GITHUB=$(curl \
+    --max-time 120 \
+    -H "Authorization: token ${_GITHUBTOKEN}" \
+    -H "Accept: application/vnd.github.v3.star+json" \
+    -sSL "https://api.github.com/$1")
+  if (echo "${_GITHUB}" | grep -q 'API rate limit exceeded'); then
+    kill -s TERM $MAIN_PID
+  else
+    echo "${_GITHUB}"
+  fi
+}
+
+# request to Github lists
+ghrequest() {
+  curl --max-time 120 -sSL "https://github.com/$1"
+}
+
 # parsing Github API for all user's starred repositories
 _PAGE=1
 
 echo "- Parsing Github API for ${_USERNAME}: page $_PAGE"
-_DATA=$(curl -H "Accept: application/vnd.github.v3.star+json" -sSL "api.github.com/users/${_USERNAME}/starred?page=${_PAGE}&per_page=100" | jq '.[] | {id: .repo.id, name: .repo.full_name, desc: .repo.description, date: .starred_at}')
+_DATA=$(ghapirequest "users/${_USERNAME}/starred?page=${_PAGE}&per_page=100" | jq '.[] | {id: .repo.id, name: .repo.full_name, desc: .repo.description, date: .starred_at}')
 _PAGE=$((_PAGE+1))
 
 while [ $_PAGE -ge 2 ]; do
   unset _NEW_DATA
   echo "- Parsing Github API for ${_USERNAME}: page $_PAGE"
-  _NEW_DATA=$(curl -H "Accept: application/vnd.github.v3.star+json" -sSL "api.github.com/users/${_USERNAME}/starred?page=${_PAGE}&per_page=100" | jq '.[] | {id: .repo.id, name: .repo.full_name, desc: .repo.description, date: .starred_at}')
+  _NEW_DATA=$(ghapirequest "users/${_USERNAME}/starred?page=${_PAGE}&per_page=100" | jq '.[] | {id: .repo.id, name: .repo.full_name, desc: .repo.description, date: .starred_at}')
   if [ "$_NEW_DATA" ]; then
     _DATA=$(echo -e "${_DATA}\n${_NEW_DATA}")
     _PAGE=$((_PAGE+1))
@@ -111,19 +144,19 @@ fi
 # parsing Github list pages (not available in the API)
 if [[ -n $_LISTNAME ]]; then
   _PAGE=1
-  
+
   echo "- Parsing Github \"${_LISTNAME}\" List for ${_USERNAME}: page $_PAGE"
-  _LIST=$(curl -sS "https://github.com/stars/${_USERNAME}/lists/${_LISTNAME}?page=1")
+  _LIST=$(ghrequest "stars/${_USERNAME}/lists/${_LISTNAME}?page=1")
   _NEXT=$(echo "${_LIST}" | pup '#user-list-repositories div div a[class="next_page"] text{}')
   _LIST=$(echo "${_LIST}" | pup '#user-list-repositories div div h3 a attr{href}')
   if [[ -n $_NEXT ]]; then
     _PAGE=$((_PAGE+1))
   fi
-  
+
   while [ $_PAGE -ge 2 ]; do
     unset _NEW_LIST _NEXT
     echo "- Parsing Github \"${_LISTNAME}\" List for ${_USERNAME}: page $_PAGE"
-    _NEW_LIST=$(curl -sS "https://github.com/stars/${_USERNAME}/lists/${_LISTNAME}?page=${_PAGE}")
+    _NEW_LIST=$(ghrequest "stars/${_USERNAME}/lists/${_LISTNAME}?page=${_PAGE}")
     _NEXT=$(echo "${_NEW_LIST}" | pup '#user-list-repositories div div a[class="next_page"] text{}')
     _NEW_LIST=$(echo "${_NEW_LIST}" | pup '#user-list-repositories div div h3 a attr{href}')
     _LIST=$(echo -e "${_LIST}\n${_NEW_LIST}")
@@ -134,7 +167,7 @@ if [[ -n $_LISTNAME ]]; then
       _PAGE=1
     fi
   done
-  
+
   _LIST=$(echo "${_LIST}" | sed -r 's,^\/,,g')
 fi
 
@@ -152,8 +185,8 @@ fi
 
 # opml file generation
 echo "- Generating OPML file Header"
-_FIRSTDATE=$(LC_ALL="C.UTF-8" TZ=GMT date -d $(echo "${_DATA}" | jq -sr 'sort_by(.date) | reverse[-1].date') '+%a, %d %b %Y %H:%M:%S %Z')
+_FIRSTDATE=$(LC_ALL="C.UTF-8" TZ=GMT date -d "$(echo "${_DATA}" | jq -sr 'sort_by(.date) | reverse[-1].date')" '+%a, %d %b %Y %H:%M:%S %Z')
-_LASTDATE=$(LC_ALL="C.UTF-8" TZ=GMT date -d $(echo "${_DATA}" | jq -sr 'sort_by(.date)[-1].date') '+%a, %d %b %Y %H:%M:%S %Z')
+_LASTDATE=$(LC_ALL="C.UTF-8" TZ=GMT date -d "$(echo "${_DATA}" | jq -sr 'sort_by(.date)[-1].date')" '+%a, %d %b %Y %H:%M:%S %Z')
 if [[ -z $_LISTNAME ]]; then
   _CATEGORY="Github - ${_USERNAME}"
 else
@@ -176,8 +209,16 @@ echo "- Generating OPML file Feeds"
 for id in $(echo "$_DATA" | jq .id); do
   _REPO_NAME=$(echo "$_DATA" | jq -r --argjson v "$id" ' . | select(.id==$v).name')
   _REPO_DESC=$(echo "$_DATA" | jq -r --argjson v "$id" ' . | select(.id==$v).desc' | sed -e 's~\&~\&amp;~g' -e 's~<~\&lt;~g' -e 's~>~\&gt;~g' -e 's~\"~\&quot;~g' -e "s~'~\&apos;~g")
+  _RELEASES=$(ghapirequest "repos/${_REPO_NAME}/releases" | jq '.[]')
+  _TYPE="releases"
+  if [[ -z "$_RELEASE" ]]; then
+    _TAGS=$(ghapirequest "repos/${_REPO_NAME}/tags" | jq '.[]')
+    if [[ -n "$_TAGS" ]]; then
+      _TYPE="tags"
+    fi
+  fi
   cat <<- EOF >> "${_FILENAME}"
-	                <outline title="${_REPO_NAME}" text="${_REPO_NAME}" type="rss" version="ATOM1" description="${_REPO_DESC}" xmlUrl="https://github.com/${_REPO_NAME}/releases.atom" htmlUrl="https://github.com/${_REPO_NAME}"></outline>
+	                <outline title="${_REPO_NAME}" text="${_REPO_NAME}" type="rss" version="ATOM1" description="${_REPO_DESC}" xmlUrl="https://github.com/${_REPO_NAME}/${_TYPE}.atom" htmlUrl="https://github.com/${_REPO_NAME}"></outline>
 EOF
 done