# -*- coding: utf-8 -*- import json import socket import sys import time import urllib.parse import urllib.request import _strptime # https://bugs.python.org/issue7980 from socket import timeout from threading import Thread from urllib.error import HTTPError, URLError import xbmc import xbmcaddon import xbmcgui import xbmcplugin from .allmusic import allmusic_artistfind from .allmusic import allmusic_artistdetails from .allmusic import allmusic_artistalbums from .discogs import discogs_artistfind from .discogs import discogs_artistdetails from .discogs import discogs_artistalbums from .fanarttv import fanarttv_artistart from .musicbrainz import musicbrainz_artistfind from .musicbrainz import musicbrainz_artistdetails from .nfo import nfo_geturl from .theaudiodb import theaudiodb_artistdetails from .theaudiodb import theaudiodb_artistalbums from .wikipedia import wikipedia_artistdetails from .utils import * ADDONID = xbmcaddon.Addon().getAddonInfo('id') ADDONNAME = xbmcaddon.Addon().getAddonInfo('name') ADDONVERSION = xbmcaddon.Addon().getAddonInfo('version') def log(txt): message = '%s: %s' % (ADDONID, txt) xbmc.log(msg=message, level=xbmc.LOGDEBUG) def get_data(url, jsonformat, retry=True): try: if url.startswith('https://musicbrainz.org/'): api_timeout('musicbrainztime') elif url.startswith('https://api.discogs.com/'): api_timeout('discogstime') headers = {} headers['User-Agent'] = '%s/%s ( http://kodi.tv )' % (ADDONNAME, ADDONVERSION) req = urllib.request.Request(url, headers=headers) resp = urllib.request.urlopen(req, timeout=5) respdata = resp.read() except URLError as e: log('URLError: %s - %s' % (e.reason, url)) return except HTTPError as e: log('HTTPError: %s - %s' % (e.reason, url)) return except socket.timeout as e: log('socket: %s - %s' % (e, url)) return if resp.getcode() == 503: log('exceeding musicbrainz api limit') if retry: xbmc.sleep(1000) get_data(url, jsonformat, retry=False) else: return elif resp.getcode() == 429: log('exceeding discogs api limit') if retry: xbmc.sleep(1000) get_data(url, jsonformat, retry=False) else: return if jsonformat: respdata = json.loads(respdata) return respdata def api_timeout(scraper): currenttime = round(time.time() * 1000) previoustime = xbmcgui.Window(10000).getProperty(scraper) if previoustime: timeout = currenttime - int(previoustime) if timeout < 1000: xbmc.sleep(1000 - timeout) xbmcgui.Window(10000).setProperty(scraper, str(round(time.time() * 1000))) class Scraper(): def __init__(self, action, key, artist, url, nfo, settings): # parse path settings self.parse_settings(settings) # this is just for backward compitability with xml based scrapers https://github.com/xbmc/xbmc/pull/11632 if action == 'resolveid': # return the result result = self.resolve_mbid(key) self.return_resolved(result) # search for artist name matches elif action == 'find': # try musicbrainz first result = self.find_artist(artist, 'musicbrainz') if result: self.return_search(result) # fallback to discogs else: result = self.find_artist(artist, 'discogs') if result: self.return_search(result) # return info using id's elif action == 'getdetails': details = {} discography = {} url = json.loads(url) artist = url.get('artist') mbartistid = url.get('mbartistid') dcid = url.get('dcid') threads = [] extrascrapers = [] discographyscrapers = [] # we have a musicbrainz id if mbartistid: scrapers = [[mbartistid, 'musicbrainz'], [mbartistid, 'theaudiodb'], [mbartistid, 'fanarttv']] for item in scrapers: thread = Thread(target = self.get_details, args = (item[0], item[1], details)) threads.append(thread) thread.start() # theaudiodb discograhy thread = Thread(target = self.get_discography, args = (mbartistid, 'theaudiodb', discography)) threads.append(thread) thread.start() # wait for musicbrainz to finish threads[0].join() # check if we have a result: if 'musicbrainz' in details: if not artist: artist = details['musicbrainz']['artist'] # scrape allmusic if we have an url provided by musicbrainz if 'allmusic' in details['musicbrainz']: extrascrapers.append([{'url': details['musicbrainz']['allmusic']}, 'allmusic']) # allmusic discograhy discographyscrapers.append([{'url': details['musicbrainz']['allmusic']}, 'allmusic']) # only scrape allmusic by artistname if explicitly enabled elif self.inaccurate and artist: extrascrapers.append([{'artist': artist}, 'allmusic']) # scrape wikipedia if we have an url provided by musicbrainz if 'wikipedia' in details['musicbrainz']: extrascrapers.append([details['musicbrainz']['wikipedia'], 'wikipedia']) elif 'wikidata' in details['musicbrainz']: extrascrapers.append([details['musicbrainz']['wikidata'], 'wikidata']) # scrape discogs if we have an url provided by musicbrainz if 'discogs' in details['musicbrainz']: extrascrapers.append([{'url': details['musicbrainz']['discogs']}, 'discogs']) # discogs discograhy discographyscrapers.append([{'url': details['musicbrainz']['discogs']}, 'discogs']) # only scrape discogs by artistname if explicitly enabled elif self.inaccurate and artist: extrascrapers.append([{'artist': artist}, 'discogs']) for item in extrascrapers: thread = Thread(target = self.get_details, args = (item[0], item[1], details)) threads.append(thread) thread.start() # get allmusic / discogs discography if we have an url for item in discographyscrapers: thread = Thread(target = self.get_discography, args = (item[0], item[1], discography)) threads.append(thread) thread.start() # we have a discogs id else: thread = Thread(target = self.get_details, args = ({'url': dcid}, 'discogs', details)) threads.append(thread) thread.start() thread = Thread(target = self.get_discography, args = ({'url': dcid}, 'discogs', discography)) threads.append(thread) thread.start() if threads: for thread in threads: thread.join() # merge discography items for site, albumlist in discography.items(): if site in details: details[site]['albums'] = albumlist else: details[site] = {} details[site]['albums'] = albumlist result = self.compile_results(details) if result: self.return_details(result) elif action == 'NfoUrl': # check if there is a musicbrainz url in the nfo file mbartistid = nfo_geturl(nfo) if mbartistid: # return the result result = self.resolve_mbid(mbartistid) self.return_nfourl(result) xbmcplugin.endOfDirectory(int(sys.argv[1])) def parse_settings(self, data): settings = json.loads(data) # note: path settings are taken from the db, they may not reflect the current settings.xml file self.bio = settings['bio'] self.discog = settings['discog'] self.genre = settings['genre'] self.lang = settings['lang'] self.mood = settings['mood'] self.style = settings['style'] self.inaccurate = settings['inaccurate'] def resolve_mbid(self, mbartistid): item = {} item['artist'] = '' item['mbartistid'] = mbartistid return item def find_artist(self, artist, site): json = True # musicbrainz if site == 'musicbrainz': url = MUSICBRAINZURL % (MUSICBRAINZSEARCH % urllib.parse.quote_plus(artist)) scraper = musicbrainz_artistfind # musicbrainz if site == 'discogs': url = DISCOGSURL % (DISCOGSSEARCH % (urllib.parse.quote_plus(artist), DISCOGSKEY , DISCOGSSECRET)) scraper = discogs_artistfind result = get_data(url, json) if not result: return artistresults = scraper(result, artist) return artistresults def get_details(self, param, site, details, discography={}): json = True # theaudiodb if site == 'theaudiodb': url = AUDIODBURL % (AUDIODBKEY, AUDIODBDETAILS % param) artistscraper = theaudiodb_artistdetails # musicbrainz elif site == 'musicbrainz': url = MUSICBRAINZURL % (MUSICBRAINZDETAILS % param) artistscraper = musicbrainz_artistdetails # fanarttv elif site == 'fanarttv': url = FANARTVURL % (param, FANARTVKEY) artistscraper = fanarttv_artistart # discogs elif site == 'discogs': # search by artistname if we do not have an url if 'artist' in param: url = DISCOGSURL % (DISCOGSSEARCH % (urllib.parse.quote_plus(param['artist']), DISCOGSKEY , DISCOGSSECRET)) artistresult = get_data(url, json) if artistresult: artists = discogs_artistfind(artistresult, param['artist']) if artists: artistresult = sorted(artists, key=lambda k: k['relevance'], reverse=True) param['url'] = artistresult[0]['dcid'] else: return else: return url = DISCOGSURL % (DISCOGSDETAILS % (param['url'], DISCOGSKEY, DISCOGSSECRET)) artistscraper = discogs_artistdetails # wikipedia elif site == 'wikipedia': url = WIKIPEDIAURL % param artistscraper = wikipedia_artistdetails elif site == 'wikidata': # resolve wikidata to wikipedia url result = get_data(WIKIDATAURL % param, json) try: artist = result['entities'][param]['sitelinks']['enwiki']['url'].rsplit('/', 1)[1] except: return site = 'wikipedia' url = WIKIPEDIAURL % artist artistscraper = wikipedia_artistdetails # allmusic elif site == 'allmusic': json = False # search by artistname if we do not have an url if 'artist' in param: url = ALLMUSICURL % urllib.parse.quote_plus(param['artist']) artistresult = get_data(url, json) if artistresult: artists = allmusic_artistfind(artistresult, param['artist']) if artists: param['url'] = artists[0]['url'] else: return else: return url = param['url'] artistscraper = allmusic_artistdetails result = get_data(url, json) if not result: return artistresults = artistscraper(result) if not artistresults: return details[site] = artistresults # get allmusic / discogs discography if we searched by artistname if (site == 'discogs' or site == 'allmusic') and 'artist' in param: albums = self.get_discography(param, site, {}) if albums: details[site]['albums'] = albums[site] return details def get_discography(self, param, site, discography): json = True if site == 'theaudiodb': # theaudiodb - discography albumsurl = AUDIODBURL % (AUDIODBKEY, AUDIODBDISCOGRAPHY % param) scraper = theaudiodb_artistalbums elif site == 'discogs': # discogs - discography albumsurl = DISCOGSURL % (DISCOGSDISCOGRAPHY % (param['url'], DISCOGSKEY, DISCOGSSECRET)) scraper = discogs_artistalbums elif site == 'allmusic': # allmusic - discography json = False albumsurl = param['url'] + '/discography' scraper = allmusic_artistalbums albumdata = get_data(albumsurl, json) if not albumdata: return albumresults = scraper(albumdata) if not albumresults: return discography[site] = albumresults return discography def compile_results(self, details): result = {} thumbs = [] fanart = [] extras = [] # merge metadata results, start with the least accurate sources if 'discogs' in details: for k, v in details['discogs'].items(): if v: result[k] = v if k == 'thumb' and v: thumbs.append(v) if 'wikipedia' in details: for k, v in details['wikipedia'].items(): if v: result[k] = v if 'allmusic' in details: for k, v in details['allmusic'].items(): if v: result[k] = v if k == 'thumb' and v: thumbs.append(v) if 'theaudiodb' in details: for k, v in details['theaudiodb'].items(): if v: result[k] = v if k == 'thumb' and v: thumbs.append(v) elif k == 'fanart' and v: fanart.append(v) if k == 'extras' and v: extras.append(v) if 'musicbrainz' in details: for k, v in details['musicbrainz'].items(): if v: result[k] = v if 'fanarttv' in details: for k, v in details['fanarttv'].items(): if v: result[k] = v if k == 'thumb' and v: thumbs.append(v) elif k == 'fanart' and v: fanart.append(v) if k == 'extras' and v: extras.append(v) # merge artwork from all scrapers if result: # artworks from most accurate sources first thumbs.reverse() thumbnails = [] fanart.reverse() fanarts = [] # the order for extra art does not matter extraart = [] for thumblist in thumbs: for item in thumblist: thumbnails.append(item) for extralist in extras: for item in extralist: extraart.append(item) # add the extra art to the end of the thumb list if extraart: thumbnails.extend(extraart) for fanartlist in fanart: for item in fanartlist: fanarts.append(item) # add the fanart to the end of the thumb list if fanarts: thumbnails.extend(fanarts) if thumbnails: result['thumb'] = thumbnails data = self.user_prefs(details, result) return data def user_prefs(self, details, result): # user preferences lang = 'biography' + self.lang if self.bio == 'theaudiodb' and 'theaudiodb' in details: if lang in details['theaudiodb']: result['biography'] = details['theaudiodb'][lang] elif 'biographyEN' in details['theaudiodb']: result['biography'] = details['theaudiodb']['biographyEN'] elif (self.bio in details) and ('biography' in details[self.bio]): result['biography'] = details[self.bio]['biography'] if (self.discog in details) and ('albums' in details[self.discog]): result['albums'] = details[self.discog]['albums'] if (self.genre in details) and ('genre' in details[self.genre]): result['genre'] = details[self.genre]['genre'] if (self.style in details) and ('styles' in details[self.style]): result['styles'] = details[self.style]['styles'] if (self.mood in details) and ('moods' in details[self.mood]): result['moods'] = details[self.mood]['moods'] return result def return_search(self, data): items = [] for item in data: listitem = xbmcgui.ListItem(item['artist'], offscreen=True) listitem.setArt({'thumb': item['thumb']}) listitem.setProperty('artist.genre', item['genre']) listitem.setProperty('artist.born', item['born']) listitem.setProperty('relevance', item['relevance']) if 'type' in item: listitem.setProperty('artist.type', item['type']) if 'gender' in item: listitem.setProperty('artist.gender', item['gender']) if 'disambiguation' in item: listitem.setProperty('artist.disambiguation', item['disambiguation']) url = {'artist':item['artist']} if 'mbartistid' in item: url['mbartistid'] = item['mbartistid'] if 'dcid' in item: url['dcid'] = item['dcid'] items.append((json.dumps(url), listitem, True)) if items: xbmcplugin.addDirectoryItems(handle=int(sys.argv[1]), items=items) def return_nfourl(self, item): listitem = xbmcgui.ListItem(offscreen=True) xbmcplugin.addDirectoryItem(handle=int(sys.argv[1]), url=json.dumps(item), listitem=listitem, isFolder=True) def return_resolved(self, item): listitem = xbmcgui.ListItem(path=json.dumps(item), offscreen=True) xbmcplugin.setResolvedUrl(handle=int(sys.argv[1]), succeeded=True, listitem=listitem) def return_details(self, item): if not 'artist' in item: return listitem = xbmcgui.ListItem(item['artist'], offscreen=True) if 'mbartistid' in item: listitem.setProperty('artist.musicbrainzid', item['mbartistid']) if 'genre' in item: listitem.setProperty('artist.genre', item['genre']) if 'biography' in item: listitem.setProperty('artist.biography', item['biography']) if 'gender' in item: listitem.setProperty('artist.gender', item['gender']) if 'styles' in item: listitem.setProperty('artist.styles', item['styles']) if 'moods' in item: listitem.setProperty('artist.moods', item['moods']) if 'instruments' in item: listitem.setProperty('artist.instruments', item['instruments']) if 'disambiguation' in item: listitem.setProperty('artist.disambiguation', item['disambiguation']) if 'type' in item: listitem.setProperty('artist.type', item['type']) if 'sortname' in item: listitem.setProperty('artist.sortname', item['sortname']) if 'active' in item: listitem.setProperty('artist.years_active', item['active']) if 'born' in item: listitem.setProperty('artist.born', item['born']) if 'formed' in item: listitem.setProperty('artist.formed', item['formed']) if 'died' in item: listitem.setProperty('artist.died', item['died']) if 'disbanded' in item: listitem.setProperty('artist.disbanded', item['disbanded']) if 'thumb' in item: listitem.setProperty('artist.thumbs', str(len(item['thumb']))) for count, thumb in enumerate(item['thumb']): listitem.setProperty('artist.thumb%i.url' % (count + 1), thumb['image']) listitem.setProperty('artist.thumb%i.preview' % (count + 1), thumb['preview']) listitem.setProperty('artist.thumb%i.aspect' % (count + 1), thumb['aspect']) if 'albums' in item: listitem.setProperty('artist.albums', str(len(item['albums']))) for count, album in enumerate(item['albums']): listitem.setProperty('artist.album%i.title' % (count + 1), album['title']) listitem.setProperty('artist.album%i.year' % (count + 1), album['year']) if 'musicbrainzreleasegroupid' in album: listitem.setProperty('artist.album%i.musicbrainzreleasegroupid' % (count + 1), album['musicbrainzreleasegroupid']) xbmcplugin.setResolvedUrl(handle=int(sys.argv[1]), succeeded=True, listitem=listitem)