parallized rss build
This commit is contained in:
parent
2252391159
commit
fb993d8928
5 changed files with 28 additions and 29 deletions
7
.gitignore
vendored
7
.gitignore
vendored
|
@ -1,8 +1,3 @@
|
||||||
_cache/
|
.cache/
|
||||||
_site/
|
_site/
|
||||||
_optim/
|
|
||||||
src/archive.org
|
|
||||||
.direnv/
|
.direnv/
|
||||||
_shake/
|
|
||||||
.shake/
|
|
||||||
dist-newstyle/
|
|
||||||
|
|
18
Makefile
18
Makefile
|
@ -56,17 +56,18 @@ $(HTML_INDEX): $(DST_PANDOC_FILES) $(MKINDEX)
|
||||||
ALL += $(HTML_INDEX)
|
ALL += $(HTML_INDEX)
|
||||||
|
|
||||||
# RSS
|
# RSS
|
||||||
|
|
||||||
SRC_POSTS_DIR ?= $(SRC_DIR)/posts
|
SRC_POSTS_DIR ?= $(SRC_DIR)/posts
|
||||||
SRC_POST_FILES ?= $(shell find $(SRC_POSTS_DIR) -type f -name "*$(EXT)")
|
DST_POSTS_DIR ?= $(DST_DIR)/posts
|
||||||
|
SRC_POSTS_FILES ?= $(shell find $(SRC_POSTS_DIR) -type f -name "*$(EXT)")
|
||||||
RSS_CACHE_DIR ?= $(CACHE_DIR)/rss
|
RSS_CACHE_DIR ?= $(CACHE_DIR)/rss
|
||||||
DST_RSS_FILES ?= $(subst .$(EXT),.rss, \
|
DST_RSS_FILES ?= $(patsubst %.org,%.rss, \
|
||||||
$(patsubst $(SRC_POSTS_DIR)/%,$(RSS_CACHE_DIR)/%, \
|
$(patsubst $(SRC_POSTS_DIR)/%,$(RSS_CACHE_DIR)/%, \
|
||||||
$(SRC_POSTS_FILES)))
|
$(SRC_POSTS_FILES)))
|
||||||
MK_RSS_ENTRY := ./engine/mk-rss-entry.sh
|
MK_RSS_ENTRY := ./engine/mk-rss-entry.sh
|
||||||
$(RSS_CACHE_DIR)/%.rss: $(DST_DIR)/posts/%.html $(MK_RSS_ENTRY)
|
$(RSS_CACHE_DIR)/%.rss: $(DST_POSTS_DIR)/%.html $(MK_RSS_ENTRY)
|
||||||
@mkdir -p $(RSS_CACHE_DIR)
|
@mkdir -p $(RSS_CACHE_DIR)
|
||||||
$(MK_RSS_ENTRY) $@
|
$(MK_RSS_ENTRY) "$<" "$@"
|
||||||
|
ALL += $(DST_RSS_FILES)
|
||||||
|
|
||||||
RSS := $(DST_DIR)/rss.xml
|
RSS := $(DST_DIR)/rss.xml
|
||||||
MKRSS := engine/mkrss.sh
|
MKRSS := engine/mkrss.sh
|
||||||
|
@ -74,11 +75,14 @@ $(RSS): $(DST_RSS_FILES) $(MKRSS)
|
||||||
$(MKRSS)
|
$(MKRSS)
|
||||||
ALL += $(RSS)
|
ALL += $(RSS)
|
||||||
|
|
||||||
|
rss: $(DST_RSS_FILES) $(RSS)
|
||||||
|
|
||||||
|
|
||||||
# ORG -> GEMINI
|
# ORG -> GEMINI
|
||||||
EXT := .org
|
EXT := .org
|
||||||
SRC_GMI_FILES ?= $(shell find $(SRC_DIR) -type f -name "*$(EXT)" $(NO_DRAFT))
|
SRC_GMI_FILES ?= $(shell find $(SRC_DIR) -type f -name "*$(EXT)" $(NO_DRAFT))
|
||||||
DST_GMI_FILES ?= $(subst $(EXT),.gmi, \
|
DST_GMI_FILES ?= $(subst $(EXT),.gmi, \
|
||||||
$(subst $(SRC_DIR),$(DST_DIR), \
|
$(patsubst $(SRC_DIR)/%,$(DST_DIR)/%, \
|
||||||
$(SRC_GMI_FILES)))
|
$(SRC_GMI_FILES)))
|
||||||
GMI := engine/org2gemini.sh
|
GMI := engine/org2gemini.sh
|
||||||
$(DST_DIR)/%.gmi: $(SRC_DIR)/%.org $(GMI) engine/org2gemini_step1.sh
|
$(DST_DIR)/%.gmi: $(SRC_DIR)/%.org $(GMI) engine/org2gemini_step1.sh
|
||||||
|
@ -101,6 +105,8 @@ $(GEM_ATOM): $(DST_GMI_FILES) $(MK_GEMINI_ATOM)
|
||||||
$(MK_GEMINI_ATOM)
|
$(MK_GEMINI_ATOM)
|
||||||
ALL += $(GEM_ATOM)
|
ALL += $(GEM_ATOM)
|
||||||
|
|
||||||
|
gemini: $(DST_GMI_FILES) $(GMI_INDEX) $(GEM_ATOM)
|
||||||
|
|
||||||
# Images
|
# Images
|
||||||
SRC_IMG_FILES ?= $(shell find $(SRC_DIR) -type f -name "*.jpg" -or -name "*.jpeg" -or -name "*.gif" -or -name "*.png")
|
SRC_IMG_FILES ?= $(shell find $(SRC_DIR) -type f -name "*.jpg" -or -name "*.jpeg" -or -name "*.gif" -or -name "*.png")
|
||||||
DST_IMG_FILES ?= $(patsubst $(SRC_DIR)/%,$(DST_DIR)/%, $(SRC_IMG_FILES))
|
DST_IMG_FILES ?= $(patsubst $(SRC_DIR)/%,$(DST_DIR)/%, $(SRC_IMG_FILES))
|
||||||
|
|
|
@ -145,4 +145,4 @@ template=$(< templates/post.html | perl -pe 's#(\$[^\$]*)\$#$1#g' )
|
||||||
} > "$indexfile"
|
} > "$indexfile"
|
||||||
|
|
||||||
rm -rf $tmpdir
|
rm -rf $tmpdir
|
||||||
echo "* RSS [done]"
|
echo "* HTML INDEX [done]"
|
||||||
|
|
|
@ -8,6 +8,7 @@ indexdir=".cache/rss"
|
||||||
|
|
||||||
# file to handle
|
# file to handle
|
||||||
fic="$1"
|
fic="$1"
|
||||||
|
dst="$2"
|
||||||
|
|
||||||
# RSS Metas
|
# RSS Metas
|
||||||
websiteurl="https://her.esy.fun"
|
websiteurl="https://her.esy.fun"
|
||||||
|
@ -41,15 +42,10 @@ mkcategories(){
|
||||||
|
|
||||||
autoload -U colors && colors
|
autoload -U colors && colors
|
||||||
|
|
||||||
if echo $fic|egrep -- '-(mk|min|sci|modern).html$'>/dev/null; then
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
|
|
||||||
postfile="$(echo "$fic"|sed 's#^'$postsdir'/##')"
|
postfile="$(echo "$fic"|sed 's#^'$postsdir'/##')"
|
||||||
blogfile="$(echo "$fic"|sed 's#^'$webdir'/##')"
|
blogfile="$(echo "$fic"|sed 's#^'$webdir'/##')"
|
||||||
printf "%-30s" $postfile
|
printf "%-30s" $postfile
|
||||||
xfic="$indexdir/$fic.xml"
|
xfic="${dst:r}.xml"
|
||||||
mkdir -p $(dirname $xfic)
|
|
||||||
hxclean $fic > $xfic
|
hxclean $fic > $xfic
|
||||||
d=$(finddate $xfic)
|
d=$(finddate $xfic)
|
||||||
echo -n " [$d]"
|
echo -n " [$d]"
|
||||||
|
@ -59,7 +55,6 @@ keywords=( $(findkeywords $xfic) )
|
||||||
printf ": %-55s" "$title ($keywords)"
|
printf ": %-55s" "$title ($keywords)"
|
||||||
categories=$(mkcategories $keywords)
|
categories=$(mkcategories $keywords)
|
||||||
absoluteurl="${websiteurl}/${blogfile}"
|
absoluteurl="${websiteurl}/${blogfile}"
|
||||||
dst="$indexdir/$fic.rss"
|
|
||||||
mkdir -p $(dirname $dst)
|
mkdir -p $(dirname $dst)
|
||||||
{ printf "\\n<item>"
|
{ printf "\\n<item>"
|
||||||
printf "\\n<title>%s</title>" "$title"
|
printf "\\n<title>%s</title>" "$title"
|
||||||
|
@ -68,5 +63,5 @@ mkdir -p $(dirname $dst)
|
||||||
printf "%s" "$categories"
|
printf "%s" "$categories"
|
||||||
printf "\\n<description><![CDATA[\\n%s\\n]]></description>" "$(getcontent "$xfic" "$absoluteurl")"
|
printf "\\n<description><![CDATA[\\n%s\\n]]></description>" "$(getcontent "$xfic" "$absoluteurl")"
|
||||||
printf "\\n</item>\\n\\n"
|
printf "\\n</item>\\n\\n"
|
||||||
} >> "$dst"
|
} > "$dst"
|
||||||
echo " [${fg[green]}OK${reset_color}]"
|
echo " [${fg[green]}OK${reset_color}]"
|
||||||
|
|
|
@ -32,28 +32,29 @@ formatdate() {
|
||||||
isodate() {
|
isodate() {
|
||||||
# format the date for sorting
|
# format the date for sorting
|
||||||
local d="$1"
|
local d="$1"
|
||||||
# echo "DEBUG DATE: $d" >&2
|
echo "DEBUG DATE: $d" >&2
|
||||||
LC_TIME=en_US date --date $d +'%Y-%m-%dT%H:%M:%S'
|
LC_TIME=en_US date --date "$d" +'%Y-%m-%dT%H:%M:%S'
|
||||||
}
|
}
|
||||||
|
|
||||||
finddate(){ < $1 hxselect -c $dateaccessor | sed 's/\[//g;s/\]//g;s/ .*$//' }
|
finddate(){ < $1 hxselect -c $dateaccessor }
|
||||||
|
|
||||||
autoload -U colors && colors
|
autoload -U colors && colors
|
||||||
|
|
||||||
typeset -a dates
|
typeset -a dates
|
||||||
dates=( )
|
dates=( )
|
||||||
tmpdir=$(mktemp -d)
|
tmpdir=$(mktemp -d)
|
||||||
for fic in $indexdir/*.rss; do
|
for fic in $indexdir/**/*.rss; do
|
||||||
rssdate=$(finddate $xfic)
|
rssdate=$(finddate $fic)
|
||||||
echo -n "${fic:t} [$d]"
|
echo -n "${fic:r} [$d]"
|
||||||
d=$(isodate $rssdate)
|
d=$(isodate $rssdate)
|
||||||
dates=( $d $dates )
|
dates=( $d $dates )
|
||||||
echo " [${fg[green]}OK${reset_color}]"
|
echo " [${fg[green]}OK${reset_color}]"
|
||||||
cp $fic $tmpdir/$d-${fic:t}.rss
|
cp $fic $tmpdir/$d-${${fic:h}:t}.rss
|
||||||
done
|
done
|
||||||
echo "Publishing"
|
echo "Publishing"
|
||||||
|
n=1
|
||||||
for fic in $(ls $tmpdir/*.rss | sort -r | head -n $maxarticles ); do
|
for fic in $(ls $tmpdir/*.rss | sort -r | head -n $maxarticles ); do
|
||||||
echo "${fic:t}"
|
echo "$((n++)) ${fic:t}"
|
||||||
cat $fic >> $tmpdir/rss
|
cat $fic >> $tmpdir/rss
|
||||||
done
|
done
|
||||||
|
|
||||||
|
@ -97,6 +98,8 @@ END
|
||||||
# HACK TO UPDATE OLD RSS FEEDS
|
# HACK TO UPDATE OLD RSS FEEDS
|
||||||
legacyenrss="$webdir/Scratch/en/blog/feed/feed.xml"
|
legacyenrss="$webdir/Scratch/en/blog/feed/feed.xml"
|
||||||
legacyfrrss="$webdir/Scratch/fr/blog/feed/feed.xml"
|
legacyfrrss="$webdir/Scratch/fr/blog/feed/feed.xml"
|
||||||
|
mkdir -p "${legacyenrss:h}"
|
||||||
|
mkdir -p "${legacyfrrss:h}"
|
||||||
cp -f "$rssfile" "$legacyenrss"
|
cp -f "$rssfile" "$legacyenrss"
|
||||||
cp -f "$rssfile" "$legacyfrrss"
|
cp -f "$rssfile" "$legacyfrrss"
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue