first commit with content
Signed-off-by: Tomasz Kapias <tkapias@git.tkapias.net>
This commit is contained in:
parent
db27acc239
commit
d46989a92c
191
gh-starred-to-opml.sh
Normal file
191
gh-starred-to-opml.sh
Normal file
|
|
@ -0,0 +1,191 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
shopt -s extglob
|
||||||
|
|
||||||
|
export LC_ALL="C.UTF-8"
|
||||||
|
export TZ=:/etc/localtime
|
||||||
|
|
||||||
|
Help()
|
||||||
|
{
|
||||||
|
echo "Generate an OPML 2.0 file to follow releases of starred repositories on Github"
|
||||||
|
echo "Default to all starred repos of the user, or a specific list with [-l list]."
|
||||||
|
echo
|
||||||
|
echo "Syntax: ./gh-starred-to-opml.sh [-h] -u user [-l listname] [-d date] [-o filename] [-n filename]"
|
||||||
|
echo "options:"
|
||||||
|
echo "-h Print this Help"
|
||||||
|
echo "-u string required: Github username"
|
||||||
|
echo "-l string optional: Github Stars List (url shortname)"
|
||||||
|
echo "-d string optional: ISO8601 date to filter out starred before (YYYY-mm-ddTHH:MM:SSZ)"
|
||||||
|
echo "-o string.opml optional: destination opml filename"
|
||||||
|
echo "-n string.opml Use a previous file to generate updates,"
|
||||||
|
echo " it does not require any other option,"
|
||||||
|
echo " but it uses starred_at dates, not starred in a list."
|
||||||
|
echo
|
||||||
|
}
|
||||||
|
|
||||||
|
unset -v _USERNAME _LISTNAME _DATEFILTER _FILENAME _OLDFILENAME _DATA
|
||||||
|
|
||||||
|
while getopts "hu:l:d:o:n:" option; do
|
||||||
|
case $option in
|
||||||
|
h) Help; exit;;
|
||||||
|
u) _USERNAME="$OPTARG";;
|
||||||
|
l) _LISTNAME="$OPTARG";;
|
||||||
|
d) _DATEFILTER="$OPTARG";;
|
||||||
|
o) _FILENAME="$OPTARG";;
|
||||||
|
n) _OLDFILENAME="$OPTARG"
|
||||||
|
unset -v _USERNAME _LISTNAME _DATEFILTER _FILENAME _DATA;;
|
||||||
|
\?) echo -e "Unknown option: -$OPTARG \n" >&2; Help; exit 1;;
|
||||||
|
: ) echo -e "Missing argument for -$OPTARG \n" >&2; Help; exit 1;;
|
||||||
|
* ) echo -e "Unimplemented option: -$option \n" >&2; Help; exit 1;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
if [[ -n $_OLDFILENAME ]]; then
|
||||||
|
# parsing the source OPML file to generate values for the update
|
||||||
|
if [[ ! -f "${_OLDFILENAME}" ]]; then
|
||||||
|
echo 'Missing source file "${_OLDFILENAME}"' >&2
|
||||||
|
echo
|
||||||
|
Help
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
_FILENAME=${_OLDFILENAME/%*(_+([0-9]).opml)*(.opml)/_$(date '+%y%m%d%H%M').opml}
|
||||||
|
_USERNAME=$(grep "ownerName" "${_OLDFILENAME}" | sed -nr 's,.+Name>(.+)<\/ownerName.+,\1,p')
|
||||||
|
_LISTNAME=$(grep "outline" "${_OLDFILENAME}" | sed -nr 's,.+Github - (.+) - .+,\1,p')
|
||||||
|
_DATEFILTER=$(LC_ALL="C.UTF-8" date -u -d "$(grep "dateModified" "${_OLDFILENAME}" | sed -nr 's,.+dateModified>(.+)<\/dateModified.+,\1,p')" '+%Y-%m-%dT%H:%M:%S%Z')
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -z $_USERNAME ]]; then
|
||||||
|
echo 'Missing Github username' >&2
|
||||||
|
echo
|
||||||
|
Help
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -z $_FILENAME ]]; then
|
||||||
|
if [[ -z $_LISTNAME ]]; then
|
||||||
|
_FILENAME="gh_starred_${_USERNAME}.opml"
|
||||||
|
else
|
||||||
|
_FILENAME="gh_starred_${_USERNAME}_${_LISTNAME}.opml"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
|
||||||
|
_CMD_ARRAY=( curl jq pup )
|
||||||
|
for cmd in "${_CMD_ARRAY[@]}"; do
|
||||||
|
if [[ -z $(command -v $cmd) ]]; then
|
||||||
|
echo 'Requirements: $cmd could not be found' >&2
|
||||||
|
echo ' - sudo apt install curl jq' >&2
|
||||||
|
echo ' - go install github.com/ericchiang/pup@latest' >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
# parsing Github API for all user's starred repositories
|
||||||
|
_PAGE=1
|
||||||
|
|
||||||
|
echo "- Parsing Github API for ${_USERNAME}: page $_PAGE"
|
||||||
|
_DATA=$(curl -H "Accept: application/vnd.github.v3.star+json" -sSL "api.github.com/users/${_USERNAME}/starred?page=${_PAGE}&per_page=100" | jq '.[] | {id: .repo.id, name: .repo.full_name, desc: .repo.description, date: .starred_at}')
|
||||||
|
_PAGE=$((_PAGE+1))
|
||||||
|
|
||||||
|
while [ $_PAGE -ge 2 ]; do
|
||||||
|
unset _NEW_DATA
|
||||||
|
echo "- Parsing Github API for ${_USERNAME}: page $_PAGE"
|
||||||
|
_NEW_DATA=$(curl -H "Accept: application/vnd.github.v3.star+json" -sSL "api.github.com/users/${_USERNAME}/starred?page=${_PAGE}&per_page=100" | jq '.[] | {id: .repo.id, name: .repo.full_name, desc: .repo.description, date: .starred_at}')
|
||||||
|
if [ "$_NEW_DATA" ]; then
|
||||||
|
_DATA=$(echo -e "${_DATA}\n${_NEW_DATA}")
|
||||||
|
_PAGE=$((_PAGE+1))
|
||||||
|
else
|
||||||
|
echo "- Page ${_PAGE} was the last one"
|
||||||
|
_PAGE=1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "- Sorting results by name"
|
||||||
|
_DATA=$(echo "${_DATA}" | jq -s -c 'sort_by(.name) | .[]')
|
||||||
|
|
||||||
|
if [[ -n $_DATEFILTER ]]; then
|
||||||
|
echo "- Filtering out repositories starred before ${_DATEFILTER}"
|
||||||
|
_DATA=$(echo "${_DATA}" | jq -c --arg date "${_DATEFILTER}" 'select( .date > $date )')
|
||||||
|
fi
|
||||||
|
|
||||||
|
# parsing Github list pages (not available in the API)
|
||||||
|
if [[ -n $_LISTNAME ]]; then
|
||||||
|
_PAGE=1
|
||||||
|
|
||||||
|
echo "- Parsing Github \"${_LISTNAME}\" List for ${_USERNAME}: page $_PAGE"
|
||||||
|
_LIST=$(curl -sS "https://github.com/stars/${_USERNAME}/lists/${_LISTNAME}?page=1")
|
||||||
|
_NEXT=$(echo "${_LIST}" | pup '#user-list-repositories div div a[class="next_page"] text{}')
|
||||||
|
_LIST=$(echo "${_LIST}" | pup '#user-list-repositories div div h3 a attr{href}')
|
||||||
|
if [[ -n $_NEXT ]]; then
|
||||||
|
_PAGE=$((_PAGE+1))
|
||||||
|
fi
|
||||||
|
|
||||||
|
while [ $_PAGE -ge 2 ]; do
|
||||||
|
unset _NEW_LIST _NEXT
|
||||||
|
echo "- Parsing Github \"${_LISTNAME}\" List for ${_USERNAME}: page $_PAGE"
|
||||||
|
_NEW_LIST=$(curl -sS "https://github.com/stars/${_USERNAME}/lists/${_LISTNAME}?page=${_PAGE}")
|
||||||
|
_NEXT=$(echo "${_NEW_LIST}" | pup '#user-list-repositories div div a[class="next_page"] text{}')
|
||||||
|
_NEW_LIST=$(echo "${_NEW_LIST}" | pup '#user-list-repositories div div h3 a attr{href}')
|
||||||
|
_LIST=$(echo -e "${_LIST}\n${_NEW_LIST}")
|
||||||
|
if [[ -n $_NEXT ]]; then
|
||||||
|
_PAGE=$((_PAGE+1))
|
||||||
|
else
|
||||||
|
echo "- Page ${_PAGE} was the last one"
|
||||||
|
_PAGE=1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
_LIST=$(echo "${_LIST}" | sed -r 's,^\/,,g')
|
||||||
|
fi
|
||||||
|
|
||||||
|
# filtering down repositories by the List
|
||||||
|
if [[ -n $_LIST ]]; then
|
||||||
|
echo "- Filtering all starred repositories by the ones in the list."
|
||||||
|
_DATA=$(echo "${_DATA}" | jq --argjson names "$(echo "${_LIST}" | jq -R . | jq -s .)" 'select( .name as $name | $names | index($name) )')
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$(echo "${_DATA}" | jq '.name' | wc -l)" == "0" ]]; then
|
||||||
|
echo 'No entries found. Aborting.' >&2
|
||||||
|
echo
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# opml file generation
|
||||||
|
echo "- Generating OPML file Header"
|
||||||
|
_FIRSTDATE=$(LC_ALL="C.UTF-8" TZ=GMT date -d $(echo "${_DATA}" | jq -sr 'sort_by(.date) | reverse[-1].date') '+%a, %d %b %Y %H:%M:%S %Z')
|
||||||
|
_LASTDATE=$(LC_ALL="C.UTF-8" TZ=GMT date -d $(echo "${_DATA}" | jq -sr 'sort_by(.date)[-1].date') '+%a, %d %b %Y %H:%M:%S %Z')
|
||||||
|
if [[ -z $_LISTNAME ]]; then
|
||||||
|
_CATEGORY="Github - ${_USERNAME}"
|
||||||
|
else
|
||||||
|
_CATEGORY="Github - ${_LISTNAME} - ${_USERNAME}"
|
||||||
|
fi
|
||||||
|
cat <<- EOF > "${_FILENAME}"
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<opml version="2.0">
|
||||||
|
<head>
|
||||||
|
<title>${_FILENAME}</title>
|
||||||
|
<dateCreated>${_FIRSTDATE}</dateCreated>
|
||||||
|
<dateModified>${_LASTDATE}</dateModified>
|
||||||
|
<ownerName>${_USERNAME}</ownerName>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<outline text="${_CATEGORY}">
|
||||||
|
EOF
|
||||||
|
|
||||||
|
echo "- Generating OPML file Feeds"
|
||||||
|
for id in $(echo "$_DATA" | jq .id); do
|
||||||
|
_REPO_NAME=$(echo "$_DATA" | jq -r --argjson v "$id" ' . | select(.id==$v).name')
|
||||||
|
_REPO_DESC=$(echo "$_DATA" | jq -r --argjson v "$id" ' . | select(.id==$v).desc' | sed -e 's~\&~\&~g' -e 's~<~\<~g' -e 's~>~\>~g' -e 's~\"~\"~g' -e "s~'~\'~g")
|
||||||
|
cat <<- EOF >> "${_FILENAME}"
|
||||||
|
<outline title="${_REPO_NAME}" text="${_REPO_NAME}" type="rss" version="ATOM1" description="${_REPO_DESC}" xmlUrl="https://github.com/${_REPO_NAME}/releases.atom" htmlUrl="https://github.com/${_REPO_NAME}"></outline>
|
||||||
|
EOF
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "- Generating OPML file Footer"
|
||||||
|
cat <<- EOF >> "${_FILENAME}"
|
||||||
|
</outline>
|
||||||
|
</body>
|
||||||
|
</opml>
|
||||||
|
EOF
|
||||||
|
|
||||||
|
echo "- Done: Generated $(echo "${_DATA}" | jq '.name' | wc -l) entries for \"${_CATEGORY}\" in ${_FILENAME}"
|
||||||
Loading…
Reference in a new issue