#!/bin/sh
# This is a pretty evil script that's meant for
# illustrative purposes only.
# Note that the input feed is hard-coded, and the
# parsing conditions will
# only work for that feed in particular.
#
# No proxies are being used, but curl and wget do
# have proxy capabilities.
# If a variation of this script is being heavily used,
# you should probably
# take advantage of those capabilities so as to a
# void detection --- the
# script could set off alarm bells to the
# attentive webmaster.
#
# Get the URLs from the feed and put them in a
# file called urls.lst ---
# these are the URLs of the full articles.
curl http://feeds2.feedburner.com/eldis-manuals?format=html\
2>/dev/null \
grep 'a href' | cut -f 2 -d \" | sort -u | \
grep feedproxy > urls.lst
# Make sure we only process new URLs --- already
# processed URLs should
# be listed in master_urls.lst; new ones will go to
# new_urls.lst
comm -13 master_urls.lst urls.lst > new_urls.lst
##############################################################
##############################################################
##############################################################
# Use wget to scrape the content from each of the new
# urls from the feed
#
# The wget -i option causes wget to iterate through all
# the URLs listed in the
# given file, here new_urls.lst
#
# The --output-document=scraped_contents.txt option
# sends all the contents of
# each of the URLs to a file called scraped_contents.txt
# --- you can
# call it something different if you want ...
#
# The --random-wait option causes wget to wait a
# randomly chosen period of time
# before trying to get its next file. This may help
# keep your
# scraping undetected.
wget -i new_urls.lst --output-document=scraped_contents.txt\
--random-wait
##############################################################
##############################################################
##############################################################
# Get the Title of each new posting
# TODO - probably with a call to a Perl script that uses
# HTML::TreeBuilder
# Get the full text of each new posting.
# TODO - probably with a call to a Perl script that uses
# HTML::TreeBuilder
# Insert into your WP database
# TODO - quickie MySQL connection, insert appropriate
# data to appropriate WP table(s), MySQL disconnect
##############################################################
##############################################################
##############################################################
# Remember to append the new_urls.lst to the
# master_urls.lst for next time
cat new_urls.lst >> master_urls.lst
# For the comm command to work, the contents of both
# files involved must be
# sorted and have no duplicate entries
sort -u master_urls.lst > temp_urls.lst
mv temp_urls.lst master_urls.lst
exit 0
