"""Python wrapper for Google web APIs This module allows you to access Google's web APIs through SOAP, to do things like search Google and get the results programmatically. Described here: http://www.google.com/apis/ You need a Google-provided license key to use these services. Follow the link above to get one. These functions will look in several places (in this order) for the license key: - the "license_key" argument of each function - the module-level LICENSE_KEY variable (call setLicense once to set it) - an environment variable called GOOGLE_LICENSE_KEY - a file called ".googlekey" in the current directory - a file called "googlekey.txt" in the current directory - a file called ".googlekey" in your home directory - a file called "googlekey.txt" in your home directory - a file called ".googlekey" in the same directory as google.py - a file called "googlekey.txt" in the same directory as google.py Sample usage: >>> import google >>> google.setLicense('...') # must get your own key! >>> data = google.doGoogleSearch('python') >>> data.meta.searchTime 0.043221000000000002 >>> data.results[0].URL 'http://www.python.org/' >>> data.results[0].title '<b>Python</b> Language Website' See documentation of SearchResultsMetaData and SearchResult classes for other available attributes. """ __author__ = "Mark Pilgrim (f8dy@diveintomark.org)" __version__ = "0.5.2" __cvsversion__ = "$Revision$"[11:-2] __date__ = "$Date$"[7:-2] __copyright__ = "Copyright (c) 2002 Mark Pilgrim" __license__ = "Python" __credits__ = """David Ascher, for the install script Erik Max Francis, for the command line interface Michael Twomey, for HTTP proxy support""" import SOAP import os, sys, getopt LICENSE_KEY = None HTTP_PROXY = None # don't touch the rest of these constants class NoLicenseKey(Exception): pass _url = 'http://api.google.com/search/beta2' _namespace = 'urn:GoogleSearch' _false = SOAP.booleanType(0) _true = SOAP.booleanType(1) _googlefile1 = ".googlekey" _googlefile2 = "googlekey.txt" _licenseLocations = ( (lambda key: key, 'passed to the function in license_key variable'), (lambda key: LICENSE_KEY, 'module-level LICENSE_KEY variable (call setLicense to set it)'), (lambda key: os.environ.get('GOOGLE_LICENSE_KEY', None), 'an environment variable called GOOGLE_LICENSE_KEY'), (lambda key: _contentsOf(os.getcwd(), _googlefile1), '%s in the current directory' % _googlefile1), (lambda key: _contentsOf(os.getcwd(), _googlefile2), '%s in the current directory' % _googlefile2), (lambda key: _contentsOf(os.environ.get('HOME', ''), _googlefile1), '%s in your home directory' % _googlefile1), (lambda key: _contentsOf(os.environ.get('HOME', ''), _googlefile2), '%s in your home directory' % _googlefile2), (lambda key: _contentsOf(_getScriptDir(), _googlefile1), '%s in the google.py directory' % _googlefile1), (lambda key: _contentsOf(_getScriptDir(), _googlefile2), '%s in the google.py directory' % _googlefile2) ) ## administrative functions def version(): print """PyGoogle %(__version__)s %(__copyright__)s released %(__date__)s Thanks to: %(__credits__)s""" % globals() def usage(): program = os.path.basename(sys.argv[0]) print """Usage: %(program)s [options] [querytype] query options: -k, --key= <license key> Google license key (see important note below) -1, -l, --lucky show only first hit -m, --meta show meta information -r, --reverse show results in reverse order -x, --proxy= <url> use HTTP proxy -h, --help print this help -v, --version print version and copyright information -t, --test run test queries querytype: -s, --search= <query> search (default) -c, --cache= <url> retrieve cached page -p, --spelling= <word> check spelling IMPORTANT NOTE: all Google functions require a valid license key; visit http://www.google.com/apis/ to get one. %(program)s will look in these places (in order) and use the first license key it finds: * the key specified on the command line""" % vars() for get, location in _licenseLocations[2:]: print " *", location ## utility functions def setLicense(license_key): """set license key""" global LICENSE_KEY LICENSE_KEY = license_key def getLicense(license_key = None): """get license key license key can come from any number of locations; see module docs for search order""" for get, location in _licenseLocations: rc = get(license_key) if rc: return rc usage() raise NoLicenseKey, 'get a license key at http://www.google.com/apis/' def setProxy(http_proxy): """set HTTP proxy""" global HTTP_PROXY HTTP_PROXY = http_proxy def getProxy(http_proxy = None): """get HTTP proxy""" return http_proxy or HTTP_PROXY def _contentsOf(dirname, filename): filename = os.path.join(dirname, filename) if not os.path.exists(filename): return None fsock = open(filename) contents = fsock.read() fsock.close() return contents def _getScriptDir(): if __name__ == '__main__': return os.path.abspath(os.path.dirname(sys.argv[0])) else: return os.path.abspath(os.path.dirname(sys.modules[__name__].__file__)) def _marshalBoolean(value): if value: return _true else: return _false ## output formatters def makeFormatter(outputFormat): classname = "%sOutputFormatter" % outputFormat.capitalize() return globals()[classname]() def output(results, params): formatter = makeFormatter(params.get("outputFormat", "text")) outputmethod = getattr(formatter, params["func"]) outputmethod(results, params) class OutputFormatter: def boil(self, data): if type(data) == type(u""): return data.encode("ISO-8859-1", "replace") else: return data class TextOutputFormatter(OutputFormatter): def common(self, data, params): if params.get("showMeta", 0): meta = data.meta for category in meta.directoryCategories: print "directoryCategory: %s" % self.boil(category["fullViewableName"]) for attr in [node for node in dir(meta) if node <> "directoryCategories" and node[:2] <> '__']: print "%s:" % attr, self.boil(getattr(meta, attr)) def doGoogleSearch(self, data, params): results = data.results if params.get("feelingLucky", 0): results = results[:1] if params.get("reverseOrder", 0): results.reverse() for result in results: for attr in dir(result): if attr == "directoryCategory": print "directoryCategory:", self.boil(result.directoryCategory["fullViewableName"]) elif attr[:2] <> '__': print "%s:" % attr, self.boil(getattr(result, attr)) print self.common(data, params) def doGetCachedPage(self, data, params): print data self.common(data, params) doSpellingSuggestion = doGetCachedPage ## search results classes class _SearchBase: def __init__(self, params): for k, v in params.items(): if isinstance(v, SOAP.structType): v = v._asdict try: if isinstance(v[0], SOAP.structType): v = [node._asdict for node in v] except: pass self.__dict__[str(k)] = v class SearchResultsMetaData(_SearchBase): """metadata of search query results Available attributes: documentFiltering - flag indicates whether duplicate page filtering was perfomed in this search searchComments - human-readable informational message (example: "'the' is a very common word and was not included in your search") estimatedTotalResultsCount - estimated total number of results for this query estimateIsExact - flag indicates whether estimatedTotalResultsCount is an exact value searchQuery - search string that initiated this search startIndex - index of first result returned (zero-based) endIndex - index of last result returned (zero-based) searchTips - human-readable informational message on how to use Google bette directoryCategories - list of dictionaries like this: {'fullViewableName': Open Directory category, 'specialEncoding': encoding scheme of this directory category} searchTime - total search time, in seconds """ pass class SearchResult(_SearchBase): """search result Available attributes: URL - URL title - title (HTML) snippet - snippet showing query context (HTML) cachedSize - size of cached version of this result, (KB) relatedInformationPresent - flag indicates that the "related:" keyword is supported for this URL hostName: When filtering occurs, a maximum of two results from any given host is returned. When this occurs, the second resultElement that comes from that host contains the host name in this parameter. directoryCategory: dictionary like this: {'fullViewableName': Open Directory category, 'specialEncoding': encoding scheme of this directory category} directoryTitle: Open Directory title of this result (or blank) summary - Open Directory summary for this result (or blank) """ pass class SearchReturnValue: """complete search results for a single query Available attributes: meta - SearchResultsMetaData results - list of SearchResult """ def __init__(self, metadata, results): self.meta = metadata self.results = results ## main functions def doGoogleSearch(q, start=0, maxResults=10, filter=1, restrict='', safeSearch=0, language='', inputencoding='', outputencoding='', license_key = None, http_proxy = None): """search Google You need a license key to call this function; see http://www.google.com/apis/ to get one. Then you can either pass it to this function every time, or set it globally; see the module docs for details. Parameters: q - search string. Anything you could type at google.com, you can pass here. See http://www.google.com/help/features.html for examples of advanced features. start (optional) - zero-based index of first desired result (for paging through multiple pages of results) maxResults (optional) - maximum number of results, currently capped at 10 filter (optional) - set to 1 to filter out similar results, set to 0 to see everything restrict (optional) - restrict results by country or topic. Examples: Ukraine - search only sites located in Ukraine linux - search Linux sites only mac - search Mac sites only bsd - search FreeBSD sites only See the APIs_reference.html file in the SDK (http://www.google.com/apis/download.html) for more advanced examples and a full list of country codes and topics. safeSearch (optional) - set to 1 to filter results with SafeSearch (no adult material) language (optional) - restricts search to documents in one or more languages. Example: lang_en - only return pages in English lang_fr - only return pages in French See the APIs_reference.html file in the SDK (http://www.google.com/apis/download.html) for more advanced examples and a full list of language codes. inputencoding (optional) - sets the character encoding of q parameter outputencoding (optional) - sets the character encoding of the returned results See the APIs_reference.html file in the SDK (http://www.google.com/apis/download.html) for a full list of encodings. http_proxy (optional) - address of HTTP proxy to use for sending and receiving SOAP messages Returns: SearchReturnValue .meta - SearchMetaData .results - list of SearchResult See documentation of these individual classes for list of available attributes """ http_proxy = getProxy(http_proxy) remoteserver = SOAP.SOAPProxy(_url, namespace=_namespace, http_proxy=http_proxy) license_key = getLicense(license_key) filter = _marshalBoolean(filter) safeSearch = _marshalBoolean(safeSearch) data = remoteserver.doGoogleSearch(license_key, q, start, maxResults, filter, restrict, safeSearch, language, inputencoding, outputencoding) metadata = data._asdict del metadata["resultElements"] metadata = SearchResultsMetaData(metadata) results = [SearchResult(node._asdict) for node in data.resultElements] return SearchReturnValue(metadata, results) def doGetCachedPage(url, license_key = None, http_proxy = None): """get page from Google cache You need a license key to call this function; see http://www.google.com/apis/ to get one. Then you can either pass it to this function every time, or set it globally; see the module docs for details. Parameters: url - address of page to get license_key (optional) - Google license key http_proxy (optional) - address of HTTP proxy to use for sending and receiving SOAP messages Returns: string, text of cached page """ http_proxy = getProxy(http_proxy) remoteserver = SOAP.SOAPProxy(_url, namespace=_namespace, http_proxy=http_proxy) license_key = getLicense(license_key) return remoteserver.doGetCachedPage(license_key, url) def doSpellingSuggestion(phrase, license_key = None, http_proxy = None): """get spelling suggestions from Google You need a license key to call this function; see http://www.google.com/apis/ to get one. Then you can either pass it to this function every time, or set it globally; see the module docs for details. Parameters: phrase - word or phrase to spell-check http_proxy (optional) - address of HTTP proxy to use for sending and receiving SOAP messages Returns: text of suggested replacement, or None """ http_proxy = getProxy(http_proxy) remoteserver = SOAP.SOAPProxy(_url, namespace=_namespace, http_proxy=http_proxy) license_key = getLicense(license_key) return remoteserver.doSpellingSuggestion(license_key, phrase) ## functional test suite (see googletest.py for unit test suite) def test(): try: getLicense(None) except NoLicenseKey: return print "Searching for Python at google.com..." data = doGoogleSearch("Python") output(data, {"func": "doGoogleSearch"}) print "\nSearching for 5 _French_ pages about Python, encoded in ISO-8859-1..." data = doGoogleSearch("Python", language='lang_fr', outputencoding='ISO-8859-1', maxResults=5) output(data, {"func": "doGoogleSearch"}) phrase = "Pyhton programming languager" print "\nTesting spelling suggetions for '%s'..." % phrase data = doSpellingSuggestion(phrase) output(data, {"func": "doSpellingSuggestion"}) ## main driver for command-line use def main(argv): if not argv: usage() return q = None func = None http_proxy = None license_key = None feelingLucky = 0 showMeta = 0 reverseOrder = 0 runTest = 0 outputFormat = "text" try: opts, args = getopt.getopt(argv, "s:c:p:k:lmrx:hvt1", ["search=", "cache=", "spelling=", "key=", "lucky", "meta", "reverse", "proxy=", "help", "version", "test"]) except getopt.GetoptError: usage() sys.exit(2) for opt, arg in opts: if opt in ("-s", "--search"): q = arg func = "doGoogleSearch" elif opt in ("-c", "--cache"): q = arg func = "doGetCachedPage" elif opt in ("-p", "--spelling"): q = arg func = "doSpellingSuggestion" elif opt in ("-k", "--key"): license_key = arg elif opt in ("-l", "-1", "--lucky"): feelingLucky = 1 elif opt in ("-m", "--meta"): showMeta = 1 elif opt in ("-r", "--reverse"): reverseOrder = 1 elif opt in ("-x", "--proxy"): http_proxy = arg elif opt in ("-h", "--help"): usage() elif opt in ("-v", "--version"): version() elif opt in ("-t", "--test"): runTest = 1 if runTest: setLicense(license_key) setProxy(http_proxy) test() if args and not q: q = args[0] func = "doGoogleSearch" if func: results = globals()[func](q, http_proxy=http_proxy, license_key=license_key) output(results, locals()) if __name__ == '__main__': main(sys.argv[1:]) # vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78: