#! /usr/bin/python # -*- coding: iso-8859-15 -*- ## ## importmodule_vibez_privatedb.py ## - "privatedb" import module for It'sGotTheVibez, an last.fm offline ## scrobbler for Trekstor's "Vibez" portable music player. ## ## This import module supports Vibez firmwares up to version 1.15 ## by reading track listening data from the Vibez' ".private/smalldb" ## file. ## From firmware versions 1.15 on you should use the "musiclog" import ## module which provides much more accurate and robust scrobbling. ## ## See http://www.ohrner.net/ for latest news and updates, please. ## ## $Id$ ## $URL$ ## ## Copyright (C) 2007-2008 Gunter Ohrner "gunter _(@)_ ohrner.net" ## ## This program is free software; you can redistribute it and/or modify ## it under the terms of the GNU General Public License as published by ## the Free Software Foundation; either version 2 of the License, or ## (at your option) any later version. ## ## This program is distributed in the hope that it will be useful, ## but WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ## GNU General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with this program; if not, write to the Free Software ## Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ## import math import os import itsgotthevibezlib import lastfmsubmitter import time import utilitylib import vibezdb debug = True debug = False ################################################################# ### Public Import Module API ################################################################# IMPORT_MODULE_NAME = 'privatedb' def publishOptions(option_parser, default_config_value_dict): '''Announce extra application options / parameters to provide use with the data we need.''' parser = option_parser default_config_value_dict['vibez_db'] = 'smalldb' parser.add_option( '-z', '--vibez-db', dest='vibez_db', help=('[%s] Path/filename of your Trekstor Vibez\' "smalldb" file. ' 'The default value is "%s".' % (IMPORT_MODULE_NAME, default_config_value_dict['vibez_db'])), metavar='DBFILE') def initImportModule(scrobbler_cfg, persistent_state): '''Check settings and environment and initialize module if everything is ok.''' if persistent_state is None: persistent_state = ImPersistentStateImpl() runstate = RuntimeState() ## update runtime state (not persistent between runs) checkAndPrepareRunEnvironment(scrobbler_cfg, runstate) runstate.db = openVibezDb(runstate.db_filename) return (runstate, persistent_state) def importListeningEvents(scrobbler_cfg, runstate, persistent_state): '''Actually import the listening data.''' if scrobbler_cfg.remove_pending_events: resyncScrobblerState(persistent_state, runstate.db.records) return [] else: return processDb(persistent_state, runstate.db) ################################################################# ## Private Module Implementation ################################################################# def checkAndPrepareRunEnvironment(scrobbler_cfg, runtime_state): if scrobbler_cfg.vibez_db is None: raise itsgotthevibezlib.ItsGotTheVibezConfigurationException( 'No database file was specified. The private DB import ' 'module must know where to find the database.') runtime_state.db_filename = os.path.expanduser(scrobbler_cfg.vibez_db) if not os.path.exists(runtime_state.db_filename): raise itsgotthevibezlib.ItsGotTheVibezRuntimeException( 'The Vibez\' db file "%s" doesn\'t exist!' % runtime_state.db_filename) class RuntimeState( object ): def __init__(self): self.db_filename = None self.db = None class ImPersistentStateImpl( itsgotthevibezlib.AbstractImportModuleState ): def __init__(self): itsgotthevibezlib.AbstractImportModuleState.__init__(self, IMPORT_MODULE_NAME) ## Import module state object version identifier self.version = 1 ## dict of TrackPlayState objects, indexed by ## uppercased file name ## This is the known state to start with, ## these tracks already have been submitted to ## last.fm that often, or at least are already ## queued in the agenda-list. self.known_tracks = {} class TrackPlayState: """A track's 'play state' - ie. a counter representing the number of times the track has been played so far and a timestamp representing the last time the track has been listened to. (timestamp representing start-of-track as a Unix timestamp in local time) WARNING: Changing this into a new-style-class will drop state file compatibility! """ def __init__(self, track_id, listen_count, listen_timestamp): self.id = track_id self.lcount = listen_count self.ltime = listen_timestamp class PlaySlot: def __init__(self, start, length): self.start = start self.length = length def openVibezDb(db_filename): """Read the Vibez's DB""" try: import mmap db_size = os.stat(db_filename).st_size db_fd = os.open(db_filename, os.O_RDONLY) db_str = mmap.mmap(db_fd, db_size, access = mmap.ACCESS_READ) except ImportError: ## IronPython and JPython have no mmap db_file = None try: db_file = open(db_filename) db_str = db_file.read() finally: utilitylib.safeClose(db_file) return vibezdb.VibezDb(db_str) def positionPendingListensIntoSlots(slots, unsubmitted_plays, usp_idx, vibez_recs_by_key, slot_assignments): '''Slightly hairy recursive method which tries to find "good" playing time positions for listened tracks. Probably overly complicated.''' finished = False if usp_idx == len(unsubmitted_plays): finished = True else: track_state = unsubmitted_plays[usp_idx] vibez_record = vibez_recs_by_key[track_state.id] finished = track_state.lcount == vibez_record.lcount if finished: ## nothing to do return utilitylib.flattenListOfLists(slot_assignments.values()) tsr = None for slot in slots: track_duration = math.floor(vibez_record.durationms/1000.0) if debug: print 'Slot-Start:', slot.start print 'Slot-len:', slot.length print 'Last submitted tracktime:', track_state.ltime print 'Latest known track time:', vibez_record.ltime print 'track duration (s):', track_duration ## Slot length suffices and slot lies in the time range in which the ## user listened to this track? if slot.length >= track_duration \ and track_state.ltime <= slot.start < vibez_record.ltime: ## Compute listening time at end of slot (slotlen - tracklen) slot.length -= track_duration tsr = getSubmitRecordForVibezRecord(vibez_record, ## Position listening at end of ## slot, slot.length has been ## updated in the prev. line! slot.start + slot.length) slot_assignments.setdefault(slot, []).append(tsr) track_state.lcount += 1 ## finished usp_idx? incr_usp_idx = (track_state.lcount == vibez_record.lcount) assignments = positionPendingListensIntoSlots(slots, unsubmitted_plays, usp_idx + incr_usp_idx, vibez_recs_by_key, slot_assignments) if assignments is not None: ## Our children finished successfully, so do we! :-) return assignments else: ## Children where not able to find a valid track-to-slot assignment ## So undo our changes and try the next matching slot. track_state.lcount -= 1 slot_assignments[slot].remove(-1) slot.length += track_duration ## If we land here, we were not able to find a valid track-to-slot- ## assignment. :-( ## Notify our parent. return None def getListenStartTime(vibez_rec): """The Vibez record's ltime field stores the time a track *finished* playing as a unix timestamp, the durationms field contains the track's length in milliseconds. This method computes the time the provided track *started* playing, as required by the audioscrobbler system.""" return vibez_rec.ltime - math.ceil(vibez_rec.durationms/1000.0) def getKeyForVibezRecord(rec): """Compute an index key for the provided Vibez record, currently the record's path-/filename, converted to uppercase letters, is used.""" return rec.filename.upper() def getSubmitRecordForVibezRecord(vibez_rec, ltime = None): '''Convert a Vibez track record as returned by vibezdb.py into an AudioScrobbler track submit record as required by lastfmsubmitter.py If set, ltime is the time a track STARTED playing, in contrast of how the Vibez stores this timestamp.''' if ltime is None: ltime = getListenStartTime(vibez_rec) duration_sec = math.floor(vibez_rec.durationms/1000.0) return lastfmsubmitter.TrackSubmitRecord( vibez_rec.artist, vibez_rec.title, duration_sec, ltime, duration_sec, None, ## Musicbrainz ID vibez_rec.trackno, vibez_rec.album, None, lastfmsubmitter.TrackSubmitRecord.SOURCE_UNKNWON) def isRecValid(vibez_rec): '''Return True if the provided Vibez track record looks complete''' return vibez_rec.type != '' and vibez_rec.artist != '' \ and vibez_rec.title != '' def createTrackSubmitRecordQueue(base_time, scrobbler_state, vibez_records): """Process information gathered from the vibez and the known set of play counters to reconstruct a possible listening sequence of the tracks. Build a list of track submit records from this information and return it. 1) We first process all listening event we know the exact listening time for. (The last listening event of each track.) 2) Then we try to find a possible sequence in which all other listenings may have occured. We even try to find matching time slots between the known listening times where songs may belong, but after all it's just guesswork... """ ## First, we build an initial list of TrackSubmitRecords ## for all plays where we know the exact time stamps. unsubmitted_plays = [] vibez_recs_by_key = {} ## Re-create known_tracks dictionary to get rid of all records ## for which the corresponding vibez record has been deleted. track_states = scrobbler_state.known_tracks scrobbler_state.known_tracks = {} submit_list = [] for rec in vibez_records: if not isRecValid(rec): continue key = getKeyForVibezRecord(rec) if key not in track_states: track_state = TrackPlayState(key, 0, base_time) else: track_state = track_states[key] ## re-add seen track_state record scrobbler_state.known_tracks[key] = track_state if debug: print 'track_state.lcount:', track_state.lcount, \ 'rec.lcount:', rec.lcount ## If we know about more plays than the Vibez, its play ## counter probably had been reset in the meantime, so ## we do the same. if track_state.lcount > rec.lcount: if debug: print 'Resetting counter for track: ', key track_state.lcount = 0 ## Has the track been listened to? if track_state.lcount < rec.lcount: track_state.lcount += 1 submit_list += [ getSubmitRecordForVibezRecord(rec) ] if track_state.lcount < rec.lcount: ## Remember pending listen count and previous listening time stamp. ## All further listens must have happened after this time stamp and ## before the ltime stored in the vibez DB record. unsubmitted_plays += [ track_state ] vibez_recs_by_key[key] = rec if debug: print 'len(unsubmitted_plays):', len(unsubmitted_plays) ## Still some work to do... ## build list of available empty "slots" ## A slot is an "unoccupied" chunk of time in which no track ## is known to be played. It can later be used to position ## listening events into of which the listening time is not ## known, thus being a means to synthesize a "plausible" ## listening time. ## We'll iterate over all new track submissions we've already ## found and chop all known listening events out of the time ## line between the earliest known listening time of the last ## synchronisation and the latest listening time we just read ## from the DB ("now"). ## The remaining holes will be the available time slots we're ## looking for. submit_list.sort(lambda tsr1, tsr2: cmp(tsr1.play_timestamp, tsr2.play_timestamp) ) slots = [] slot_start = base_time for tsr in submit_list: slot_len = tsr.play_timestamp - slot_start ## Possibly adjust listening time and compute playing duration. ## This will change the original values if slot_len is negative, ## ie. there was insufficient time to listen to this track in ## full. tsr.play_timestamp = max(tsr.play_timestamp, tsr.play_timestamp - slot_len) played_duration = min(tsr.track_duration, tsr.track_duration + slot_len) if debug: print tsr.title, 'TS: %d' % tsr.play_timestamp, \ utilitylib.formatTS(tsr.play_timestamp), \ 'Dur: %d' % tsr.track_duration print 'new slot, len:', slot_len if slot_len > 2: ## Remember free slots longer than 2 seconds. slots += [ PlaySlot(slot_start, slot_len) ] elif slot_len < 0: ## Track has been seeked in while playing. ## Update TrackSubmitRecord accordingly. ## It's lastfmsubmitter.py's task to enforce any last.fm ## protocol specific filtering policies for tracks which have ## been skipped while being played. print 'Warning: Time slot is too short (%ds), track (%ds) has been seeked in while playing (%ds).' % (slot_len, tsr.track_duration, played_duration) tsr.playing_duration = played_duration ## Prepare next slot, this is the end time of the current slot and ## at the same time the starting time of the next one. slot_start = tsr.play_timestamp + played_duration ## finish, if possible ## This is positioned here so the submit_list cleanup (removal of ## incompletely played tracks) is done first. if len(unsubmitted_plays) == 0: return submit_list ## fill slots backwards slots.reverse() ## compute slot assignment submit_list = positionPendingListensIntoSlots(slots, unsubmitted_plays, 0, vibez_recs_by_key, { None: submit_list }) if submit_list is not None: submit_list.sort(lambda tsr1, tsr2: cmp(tsr1.play_timestamp, tsr2.play_timestamp) ) return submit_list def resyncScrobblerState(scrobbler_state, vibez_records): """Update all track counters remembered in the scrobbler state file to match the track counts stored on the Vibez, without submitting anything.""" track_states = scrobbler_state.known_tracks ## Re-create known_tracks dictionary to get rid of all records ## for which the corresponding vibez record has been deleted. scrobbler_state.known_tracks = {} for rec in vibez_records: if not isRecValid(rec): continue key = getKeyForVibezRecord(rec) if key not in track_states: track_state = TrackPlayState(key, rec.lcount, rec.ltime) else: track_state = track_states[key] track_state.lcount = rec.lcount scrobbler_state.known_tracks[key] = track_state print 'State file contains %d track states after resync.' \ % len(scrobbler_state.known_tracks) def processDb(scrobbler_state, db): if len(scrobbler_state.known_tracks) > 0: base_time = max([ state.ltime for state in scrobbler_state.known_tracks.values() ]) + 1 else: base_time = 0 ## read data base and create list of to-be-submitted track listens submit_records = createTrackSubmitRecordQueue(base_time, scrobbler_state, db.records) print 'Managing state of %d track entries.' \ % len(scrobbler_state.known_tracks) if submit_records is None: raise itsgotthevibezlib.ItsGotTheVibezInternalException( 'Some tracks could not be positioned!') ## Fix timestamps in Submit Records. for tsr in submit_records: ## We must fix the timezone at this point. The Vibez usually ## stores the timestamp in local time, not UTC, but last.fm expects ## UTC. ## FIXME: Doesn't work correctly! tsr.play_timestamp = time.mktime(time.gmtime(tsr.play_timestamp)) ## debug if debug: print tsr.title, ';', tsr.play_timestamp, ';', \ utilitylib.formatTS(tsr.play_timestamp) ## update agenda... return submit_records