Python Craigslist Search Script Version 2

Massive revisions to my script. Not it will remember the locations you searched in that session. It also searches specific sections. The script will stay open and loop until you close it.

Download CLget version 2

import re
import datetime
import time
import string
import urllib
import urllib2

results = re.compile('', re.DOTALL)

#results = re.compile('

.+

sort by') delay = 100 def search_all(): t = datetime.datetime.now() tyme = time.mktime(t.timetuple()) for city in list(set(cities)): url = "http://" + city + ".craigslist.org/search/?areaID=126&subAreaID=&query=" + query.replace(' ', ',') + "&catAbb=sss" #Setup headers to spoof Mozilla dat = None ua = "Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US; rv:1.9.1.4) Gecko/20091007 Firefox/3.5.4" head = {'User-agent': ua} errorcount = 0 #Get page req = urllib2.Request(url, dat, head) try: response = urllib2.urlopen(req) except urllib2.HTTPError, e: if errorcount < 1: errorcount = 1 print "Request failed, retrying in " + str(delay) + " seconds" time.sleep(int(delay)) response = urllib2.urlopen(req) except urllib2.URLError, e: if errorcount < 1: errorcount = 1 print "Request failed, retrying in " + str(delay) + " seconds" time.sleep(int(delay)) response = urllib2.urlopen(req) msg = response.read() errorcount = 0 res = results.findall(msg) res = str(res) res = res.replace('[', '') res = res.replace(']', '') res = res.replace(chr(92)+'t', chr(9)) res = res.replace(chr(92)+'n', chr(13)) res = res.replace(chr(92)+'r', chr(13)) res = res.replace(chr(39), '') outp = open("results" + str(tyme) + ".html", "a") outp.write(city) outp.write(str(res)) outp.close() def search_section(): t = datetime.datetime.now() tyme = time.mktime(t.timetuple()) for city in list(set(cities)): for section in list(set(sections)): url = "http://" + city + ".craigslist.org/search/" + section + "?query=" + query.replace(' ', ',') #Setup headers to spoof Mozilla dat = None ua = "Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US; rv:1.9.1.4) Gecko/20091007 Firefox/3.5.4" head = {'User-agent': ua} errorcount = 0 #Get page req = urllib2.Request(url, dat, head) try: response = urllib2.urlopen(req) except urllib2.HTTPError, e: if errorcount < 1: errorcount = 1 print "Request failed, retrying in " + str(delay) + " seconds" time.sleep(int(delay)) response = urllib2.urlopen(req) except urllib2.URLError, e: if errorcount < 1: errorcount = 1 print "Request failed, retrying in " + str(delay) + " seconds" time.sleep(int(delay)) response = urllib2.urlopen(req) msg = response.read() errorcount = 0 res = results.findall(msg) res = str(res) res = res.replace('[', '') res = res.replace(']', '') res = res.replace(chr(92)+'t', chr(9)) res = res.replace(chr(92)+'n', chr(13)) res = res.replace(chr(92)+'r', chr(13)) res = res.replace(chr(39), '') outp = open("results" + str(tyme) + ".html", "a") outp.write(city) outp.write(str(res)) outp.close() print "Welcome to CLget!" print "" print "Inputs:" print "Location - enter the area you want to search" print "\tEnter multiple areas separated by ONLY commas" print "Query - enter query as you normally would" print "Section - enter the 3 letter section code you want to search" print "\tEnter multiple sections separated by ONLY commas" print "\n\nLeave the Location blank to use the location(s) last entered" print "in this session of CLget. The script also remembers the last" print "entered sections you searched." while 1: print "\n\n" cityIN = raw_input('Location: ') if not cityIN: cityIN = cityINold cityINold = cityIN query = raw_input("Query: ") s = ',' cities = re.split(s,cityIN) sectionIN = raw_input('Section: ') sections = re.split(s,sectionIN) if not sectionIN: search_all() else: search_section()
This entry was posted in Python and tagged , , , , , , , . Bookmark the permalink.

3 Responses to Python Craigslist Search Script Version 2

  1. zerodtkjoe says:

    Thanks for the info

  2. Dave says:

    I have modified your example a bit, and have added some features. I can post if interested? I am in the process of adding a radius search of a “home” city.

  3. Calvin says:

    You’re free to use it however you please and re-post the code.

Leave a Reply

Your email address will not be published. Required fields are marked *