/***************************************************************************
* __________ __ ___.
* Open \______ \ ____ ____ | | _\_ |__ _______ ___
* Source | _// _ \_/ ___\| |/ /| __ \ / _ \ \/ /
* Jukebox | | ( <_> ) \___| < | \_\ ( <_> > < <
* Firmware |____|_ /\____/ \___ >__|_ \|___ /\____/__/\_ \
* \/ \/ \/ \/ \/
* $Id$
*
* Copyright (C) 2005 by Miika Pekkarinen
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
* KIND, either express or implied.
*
****************************************************************************/
/*
* TagCache API
*
* ----------x---------x------------------x-----
* | | | External
* +---------------x-------+ | TagCache | Libraries
* | Modification routines | | Core |
* +-x---------x-----------+ | |
* | (R/W) | | | |
* | +------x-------------x-+ +-------------x-----+ |
* | | x==x Filters & clauses | |
* | | Search routines | +-------------------+ |
* | | x============================x DirCache
* | +-x--------------------+ | (optional)
* | | (R) |
* | | +-------------------------------+ +---------+ |
* | | | DB Commit (sort,unique,index) | | | |
* | | +-x--------------------------x--+ | Control | |
* | | | (R/W) | (R) | Thread | |
* | | | +----------------------+ | | | |
* | | | | TagCache DB Builder | | +---------+ |
* | | | +-x-------------x------+ | |
* | | | | (R) | (W) | |
* | | | | +--x--------x---------+ |
* | | | | | Temporary Commit DB | |
* | | | | +---------------------+ |
* +-x----x-------x--+ |
* | TagCache RAM DB x==\(W) +-----------------+ |
* +-----------------+ \===x | |
* | | | | (R) | Ram DB Loader x============x DirCache
* +-x----x---x---x---+ /==x | | (optional)
* | Tagcache Disk DB x==/ +-----------------+ |
* +------------------+ |
*
*/
/* #define LOGF_ENABLE */
#include <stdio.h>
#include <stdlib.h>
#include <ctype.h>
#include "config.h"
#include "ata_idle_notify.h"
#include "thread.h"
#include "kernel.h"
#include "system.h"
#include "logf.h"
#include "string.h"
#include "usb.h"
#include "metadata.h"
#include "metadata.h"
#include "tagcache.h"
#include "buffer.h"
#include "crc32.h"
#include "misc.h"
#include "settings.h"
#include "dir.h"
#include "structec.h"
#include "tagcache.h"
#ifndef __PCTOOL__
#include "splash.h"
#include "lang.h"
#include "eeprom_settings.h"
#endif
#ifdef __PCTOOL__
#define yield() do { } while(0)
#define sim_sleep(timeout) do { } while(0)
#define do_timed_yield() do { } while(0)
#endif
#ifndef __PCTOOL__
/* Tag Cache thread. */
static struct event_queue tagcache_queue;
static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)];
static const char tagcache_thread_name[] = "tagcache";
#endif
#define UNTAGGED "<Untagged>"
/* Previous path when scanning directory tree recursively. */
static char curpath[TAG_MAXLEN+32];
/* Used when removing duplicates. */
static char *tempbuf; /* Allocated when needed. */
static long tempbufidx; /* Current location in buffer. */
static long tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */
static long tempbuf_left; /* Buffer space left. */
static long tempbuf_pos;
/* Tags we want to get sorted (loaded to the tempbuf). */
static const int sorted_tags[] = { tag_artist, tag_album, tag_genre,
tag_composer, tag_comment, tag_albumartist, tag_grouping, tag_title };
/* Uniqued tags (we can use these tags with filters and conditional clauses). */
static const int unique_tags[] = { tag_artist, tag_album, tag_genre,
tag_composer, tag_comment, tag_albumartist, tag_grouping };
/* Numeric tags (we can use these tags with conditional clauses). */
static const int numeric_tags[] = { tag_year, tag_discnumber,
tag_tracknumber, tag_length, tag_bitrate, tag_playcount, tag_rating,
tag_playtime, tag_lastplayed, tag_commitid, tag_mtime,
tag_virt_length_min, tag_virt_length_sec,
tag_virt_playtime_min, tag_virt_playtime_sec,
tag_virt_entryage, tag_virt_autoscore };
/* String presentation of the tags defined in tagcache.h. Must be in correct order! */
static const char *tags_str[] = { "artist", "album", "genre", "title",
"filename", "composer", "comment", "albumartist", "grouping", "year",
"discnumber", "tracknumber", "bitrate", "length", "playcount", "rating",
"playtime", "lastplayed", "commitid", "mtime" };
/* Status information of the tagcache. */
static struct tagcache_stat tc_stat;
/* Queue commands. */
enum tagcache_queue {
Q_STOP_SCAN = 0,
Q_START_SCAN,
Q_IMPORT_CHANGELOG,
Q_UPDATE,
Q_REBUILD,
/* Internal tagcache command queue. */
CMD_UPDATE_MASTER_HEADER,
CMD_UPDATE_NUMERIC,
};
struct tagcache_command_entry {
int32_t command;
int32_t idx_id;
int32_t tag;
int32_t data;
};
static struct tagcache_command_entry command_queue[TAGCACHE_COMMAND_QUEUE_LENGTH];
static volatile int command_queue_widx = 0;
static volatile int command_queue_ridx = 0;
static struct mutex command_queue_mutex;
/* Tag database structures. */
/* Variable-length tag entry in tag files. */
struct tagfile_entry {
short tag_length; /* Length of the data in bytes including '\0' */
short idx_id; /* Corresponding entry location in index file of not unique tags */
char tag_data[0]; /* Begin of the tag data */
};
/* Fixed-size tag entry in master db index. */
struct index_entry {
int32_t tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */
int32_t flag; /* Status flags */
};
/* Header is the same in every file. */
struct tagcache_header {
int32_t magic; /* Header version number */
int32_t datasize; /* Data size in bytes */
int32_t entry_count; /* Number of entries in this file */
};
struct master_header {
struct tagcache_header tch;
int32_t serial; /* Increasing counting number */
int32_t commitid; /* Number of commits so far */
int32_t dirty;
};
/* For the endianess correction */
static const char *tagfile_entry_ec = "ss";
/**
Note: This should be (1 + TAG_COUNT) amount of l's.
*/
static const char *index_entry_ec = "lllllllllllllllllllll";
static const char *tagcache_header_ec = "lll";
static const char *master_header_ec = "llllll";
static struct master_header current_tcmh;
#ifdef HAVE_TC_RAMCACHE
/* Header is created when loading database to ram. */
struct ramcache_header {
struct master_header h; /* Header from the master index */
struct index_entry *indices; /* Master index file content */
char *tags[TAG_COUNT]; /* Tag file content (not including filename tag) */
int entry_count[TAG_COUNT]; /* Number of entries in the indices. */
};
# ifdef HAVE_EEPROM_SETTINGS
struct statefile_header {
struct ramcache_header *hdr;
struct tagcache_stat tc_stat;
};
# endif
/* Pointer to allocated ramcache_header */
static struct ramcache_header *hdr;
#endif
/**
* Full tag entries stored in a temporary file waiting
* for commit to the cache. */
struct temp_file_entry {
long tag_offset[TAG_COUNT];
short tag_length[TAG_COUNT];
long flag;
long data_length;
};
struct tempbuf_id_list {
long id;
struct tempbuf_id_list *next;
};
struct tempbuf_searchidx {
long idx_id;
char *str;
int seek;
struct tempbuf_id_list idlist;
};
/* Lookup buffer for fixing messed up index while after sorting. */
static long commit_entry_count;
static long lookup_buffer_depth;
static struct tempbuf_searchidx **lookup;
/* Used when building the temporary file. */
static int cachefd = -1, filenametag_fd;
static int total_entry_count = 0;
static int data_size = 0;
static int processed_dir_count;
/* Thread safe locking */
static volatile int write_lock;
static volatile int read_lock;
static bool delete_entry(long idx_id);
const char* tagcache_tag_to_str(int tag)
{
return tags_str[tag];
}
bool tagcache_is_numeric_tag(int type)
{
int i;
for (i = 0; i < (int)(sizeof(numeric_tags)/sizeof(numeric_tags[0])); i++)
{
if (type == numeric_tags[i])
return true;
}
return false;
}
bool tagcache_is_unique_tag(int type)
{
int i;
for (i = 0; i < (int)(sizeof(unique_tags)/sizeof(unique_tags[0])); i++)
{
if (type == unique_tags[i])
return true;
}
return false;
}
bool tagcache_is_sorted_tag(int type)
{
int i;
for (i = 0; i < (int)(sizeof(sorted_tags)/sizeof(sorted_tags[0])); i++)
{
if (type == sorted_tags[i])
return true;
}
return false;
}
#ifdef HAVE_DIRCACHE
/**
* Returns true if specified flag is still present, i.e., dircache
* has not been reloaded.
*/
static bool is_dircache_intact(void)
{
return dircache_get_appflag(DIRCACHE_APPFLAG_TAGCACHE);
}
#endif
static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write)
{
int fd;
char buf[MAX_PATH];
int rc;
if (tagcache_is_numeric_tag(tag) || tag < 0 || tag >= TAG_COUNT)
return -1;
snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
fd = open(buf, write ? O_RDWR : O_RDONLY);
if (fd < 0)
{
logf("tag file open failed: tag=%d write=%d file=%s", tag, write, buf);
tc_stat.ready = false;
return fd;
}
/* Check the header. */
rc = ecread(fd, hdr, 1, tagcache_header_ec, tc_stat.econ);
if (hdr->magic != TAGCACHE_MAGIC || rc != sizeof(struct tagcache_header))
{
logf("header error");
tc_stat.ready = false;
close(fd);
return -2;
}
return fd;
}
static int open_master_fd(struct master_header *hdr, bool write)
{
int fd;
int rc;
fd = open(TAGCACHE_FILE_MASTER, write ? O_RDWR : O_RDONLY);
if (fd < 0)
{
logf("master file open failed for R/W");
tc_stat.ready = false;
return fd;
}
tc_stat.econ = false;
/* Check the header. */
rc = read(fd, hdr, sizeof(struct master_header));
if (hdr->tch.magic == TAGCACHE_MAGIC && rc == sizeof(struct master_header))
{
/* Success. */
return fd;
}
/* Trying to read again, this time with endianess correction enabled. */
lseek(fd, 0, SEEK_SET);
rc = ecread(fd, hdr, 1, master_header_ec, true);
if (hdr->tch.magic != TAGCACHE_MAGIC || rc != sizeof(struct master_header))
{
logf("header error");
tc_stat.ready = false;
close(fd);
return -2;
}
tc_stat.econ = true;
return fd;
}
#ifndef __PCTOOL__
static bool do_timed_yield(void)
{
/* Sorting can lock up for quite a while, so yield occasionally */
static long wakeup_tick = 0;
if (current_tick >= wakeup_tick)
{
wakeup_tick = current_tick + (HZ/4);
yield();
return true;
}
return false;
}
#endif
#if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
static long find_entry_ram(const char *filename,
const struct dirent *dc)
{
static long last_pos = 0;
int i;
/* Check if we tagcache is loaded into ram. */
if (!tc_stat.ramcache)
return -1;
if (dc == NULL)
dc = dircache_get_entry_ptr(filename);
if (dc == NULL)
{
logf("tagcache: file not found.");
return -1;
}
try_again:
if (last_pos > 0)
i = last_pos;
else
i = 0;
for (; i < hdr->h.tch.entry_count; i++)
{
if (hdr->indices[i].tag_seek[tag_filename] == (long)dc)
{
last_pos = MAX(0, i - 3);
return i;
}
do_timed_yield();
}
if (last_pos > 0)
{
last_pos = 0;
goto try_again;
}
return -1;
}
#endif
static long find_entry_disk(const char *filename)
{
struct tagcache_header tch;
static long last_pos = -1;
long pos_history[POS_HISTORY_COUNT];
long pos_history_idx = 0;
bool found = false;
struct tagfile_entry tfe;
int fd;
char buf[TAG_MAXLEN+32];
int i;
int pos = -1;
if (!tc_stat.ready)
return -2;
fd = filenametag_fd;
if (fd < 0)
{
last_pos = -1;
if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0)
return -1;
}
check_again:
if (last_pos > 0)
lseek(fd, last_pos, SEEK_SET);
else
lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
while (true)
{
pos = lseek(fd, 0, SEEK_CUR);
for (i = pos_history_idx-1; i >= 0; i--)
pos_history[i+1] = pos_history[i];
pos_history[0] = pos;
if (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
!= sizeof(struct tagfile_entry))
{
break ;
}
if (tfe.tag_length >= (long)sizeof(buf))
{
logf("too long tag #1");
close(fd);
last_pos = -1;
return -2;
}
if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
{
logf("read error #2");
close(fd);
last_pos = -1;
return -3;
}
if (!strcasecmp(filename, buf))
{
last_pos = pos_history[pos_history_idx];
found = true;
break ;
}
if (pos_history_idx < POS_HISTORY_COUNT - 1)
pos_history_idx++;
}
/* Not found? */
if (!found)
{
if (last_pos > 0)
{
last_pos = -1;
logf("seek again");
goto check_again;
}
if (fd != filenametag_fd)
close(fd);
return -4;
}
if (fd != filenametag_fd)
close(fd);
return tfe.idx_id;
}
static int find_index(const char *filename)
{
long idx_id = -1;
#if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
if (tc_stat.ramcache && is_dircache_intact())
idx_id = find_entry_ram(filename, NULL);
#endif
if (idx_id < 0)
idx_id = find_entry_disk(filename);
return idx_id;
}
bool tagcache_find_index(struct tagcache_search *tcs, const char *filename)
{
int idx_id;
if (!tc_stat.ready)
return false;
idx_id = find_index(filename);
if (idx_id < 0)
return false;
if (!tagcache_search(tcs, tag_filename))
return false;
tcs->entry_count = 0;
tcs->idx_id = idx_id;
return true;
}
static bool get_index(int masterfd, int idxid,
struct index_entry *idx, bool use_ram)
{
bool localfd = false;
if (idxid < 0)
{
logf("Incorrect idxid: %d", idxid);
return false;
}
#ifdef HAVE_TC_RAMCACHE
if (tc_stat.ramcache && use_ram)
{
if (hdr->indices[idxid].flag & FLAG_DELETED)
return false;
memcpy(idx, &hdr->indices[idxid], sizeof(struct index_entry));
return true;
}
#else
(void)use_ram;
#endif
if (masterfd < 0)
{
struct master_header tcmh;
localfd = true;
masterfd = open_master_fd(&tcmh, false);
if (masterfd < 0)
return false;
}
lseek(masterfd, idxid * sizeof(struct index_entry)
+ sizeof(struct master_header), SEEK_SET);
if (ecread(masterfd, idx, 1, index_entry_ec, tc_stat.econ)
!= sizeof(struct index_entry))
{
logf("read error #3");
if (localfd)
close(masterfd);
return false;
}
if (localfd)
close(masterfd);
if (idx->flag & FLAG_DELETED)
return false;
return true;
}
static bool write_index(int masterfd, int idxid, struct index_entry *idx)
{
/* We need to exclude all memory only flags & tags when writing to disk. */
if (idx->flag & FLAG_DIRCACHE)
{
logf("memory only flags!");
return false;
}
#ifdef HAVE_TC_RAMCACHE
/* Only update numeric data. Writing the whole index to RAM by memcpy
* destroys dircache pointers!
*/
if (tc_stat.ramcache)
{
int tag;
struct index_entry *idx_ram = &hdr->indices[idxid];
for (tag = 0; tag < TAG_COUNT; tag++)
{
if (tagcache_is_numeric_tag(tag))
{
idx_ram->tag_seek[tag] = idx->tag_seek[tag];
}
}
/* Don't touch the dircache flag or attributes. */
idx_ram->flag = (idx->flag & 0x0000ffff)
| (idx_ram->flag & (0xffff0000 | FLAG_DIRCACHE));
}
#endif
lseek(masterfd, idxid * sizeof(struct index_entry)
+ sizeof(struct master_header), SEEK_SET);
if (ecwrite(masterfd, idx, 1, index_entry_ec, tc_stat.econ)
!= sizeof(struct index_entry))
{
logf("write error #3");
logf("idxid: %d", idxid);
return false;
}
return true;
}
static bool open_files(struct tagcache_search *tcs, int tag)
{
if (tcs->idxfd[tag] < 0)
{
char fn[MAX_PATH];
snprintf(fn, sizeof fn, TAGCACHE_FILE_INDEX, tag);
tcs->idxfd[tag] = open(fn, O_RDONLY);
}
if (tcs->idxfd[tag] < 0)
{
logf("File not open!");
return false;
}
return true;
}
static bool retrieve(struct tagcache_search *tcs, struct index_entry *idx,
int tag, char *buf, long size)
{
struct tagfile_entry tfe;
long seek;
*buf = '\0';
if (tagcache_is_numeric_tag(tag))
return false;
seek = idx->tag_seek[tag];
if (seek < 0)
{
logf("Retrieve failed");
return false;
}
#ifdef HAVE_TC_RAMCACHE
if (tcs->ramsearch)
{
struct tagfile_entry *ep;
# ifdef HAVE_DIRCACHE
if (tag == tag_filename && (idx->flag & FLAG_DIRCACHE)
&& is_dircache_intact())
{
dircache_copy_path((struct dirent *)seek,
buf, size);
return true;
}
else
# endif
if (tag != tag_filename)
{
ep = (struct tagfile_entry *)&hdr->tags[tag][seek];
strncpy(buf, ep->tag_data, size-1);
return true;
}
}
#endif
if (!open_files(tcs, tag))
return false;
lseek(tcs->idxfd[tag], seek, SEEK_SET);
if (ecread(tcs->idxfd[tag], &tfe, 1, tagfile_entry_ec, tc_stat.econ)
!= sizeof(struct tagfile_entry))
{
logf("read error #5");
return false;
}
if (tfe.tag_length >= size)
{
logf("too small buffer");
return false;
}
if (read(tcs->idxfd[tag], buf, tfe.tag_length) !=
tfe.tag_length)
{
logf("read error #6");
return false;
}
buf[tfe.tag_length] = '\0';
return true;
}
static long check_virtual_tags(int tag, const struct index_entry *idx)
{
long data = 0;
switch (tag)
{
case tag_virt_length_sec:
data = (idx->tag_seek[tag_length]/1000) % 60;
break;
case tag_virt_length_min:
data = (idx->tag_seek[tag_length]/1000) / 60;
break;
case tag_virt_playtime_sec:
data = (idx->tag_seek[tag_playtime]/1000) % 60;
break;
case tag_virt_playtime_min:
data = (idx->tag_seek[tag_playtime]/1000) / 60;
break;
case tag_virt_autoscore:
if (idx->tag_seek[tag_length] == 0
|| idx->tag_seek[tag_playcount] == 0)
{
data = 0;
}
else
{
data = 100 * idx->tag_seek[tag_playtime]
/ idx->tag_seek[tag_length]
/ idx->tag_seek[tag_playcount];
}
break;
/* How many commits before the file has been added to the DB. */
case tag_virt_entryage:
data = current_tcmh.commitid - idx->tag_seek[tag_commitid] - 1;
break;
default:
data = idx->tag_seek[tag];
}
return data;
}
long tagcache_get_numeric(const struct tagcache_search *tcs, int tag)
{
struct index_entry idx;
if (!tc_stat.ready)
return false;
if (!tagcache_is_numeric_tag(tag))
return -1;
if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
return -2;
return check_virtual_tags(tag, &idx);
}
inline static bool str_ends_with(const char *str1, const char *str2)
{
int str_len = strlen(str1);
int clause_len = strlen(str2);
if (clause_len > str_len)
return false;
return !strcasecmp(&str1[str_len - clause_len], str2);
}
inline static bool str_oneof(const char *str, const char *list)
{
const char *sep;
int l, len = strlen(str);
while (*list)
{
sep = strchr(list, '|');
l = sep ? (long)sep - (long)list : (int)strlen(list);
if ((l==len) && !strncasecmp(str, list, len))
return true;
list += sep ? l + 1 : l;
}
return false;
}
static bool check_against_clause(long numeric, const char *str,
const struct tagcache_search_clause *clause)
{
if (clause->numeric)
{
switch (clause->type)
{
case clause_is:
return numeric == clause->numeric_data;
case clause_is_not:
return numeric != clause->numeric_data;
case clause_gt:
return numeric > clause->numeric_data;
case clause_gteq:
return numeric >= clause->numeric_data;
case clause_lt:
return numeric < clause->numeric_data;
case clause_lteq:
return numeric <= clause->numeric_data;
default:
logf("Incorrect numeric tag: %d", clause->type);
}
}
else
{
switch (clause->type)
{
case clause_is:
return !strcasecmp(clause->str, str);
case clause_is_not:
return strcasecmp(clause->str, str);
case clause_gt:
return 0>strcasecmp(clause->str, str);
case clause_gteq:
return 0>=strcasecmp(clause->str, str);
case clause_lt:
return 0<strcasecmp(clause->str, str);
case clause_lteq:
return 0<=strcasecmp(clause->str, str);
case clause_contains:
return (strcasestr(str, clause->str) != NULL);
case clause_not_contains:
return (strcasestr(str, clause->str) == NULL);
case clause_begins_with:
return (strcasestr(str, clause->str) == str);
case clause_not_begins_with:
return (strcasestr(str, clause->str) != str);
case clause_ends_with:
return str_ends_with(str, clause->str);
case clause_not_ends_with:
return !str_ends_with(str, clause->str);
case clause_oneof:
return str_oneof(str, clause->str);
default:
logf("Incorrect tag: %d", clause->type);
}
}
return false;
}
static bool check_clauses(struct tagcache_search *tcs,
struct index_entry *idx,
struct tagcache_search_clause **clause, int count)
{
int i;
#ifdef HAVE_TC_RAMCACHE
if (tcs->ramsearch)
{
/* Go through all conditional clauses. */
for (i = 0; i < count; i++)
{
struct tagfile_entry *tfe;
int seek;
char buf[256];
char *str = NULL;
seek = check_virtual_tags(clause[i]->tag, idx);
if (!tagcache_is_numeric_tag(clause[i]->tag))
{
if (clause[i]->tag == tag_filename)
{
retrieve(tcs, idx, tag_filename, buf, sizeof buf);
str = buf;
}
else
{
tfe = (struct tagfile_entry *)&hdr->tags[clause[i]->tag][seek];
str = tfe->tag_data;
}
}
if (!check_against_clause(seek, str, clause[i]))
return false;
}
}
else
#endif
{
/* Check for conditions. */
for (i = 0; i < count; i++)
{
struct tagfile_entry tfe;
int seek;
char str[256];
seek = check_virtual_tags(clause[i]->tag, idx);
memset(str, 0, sizeof str);
if (!tagcache_is_numeric_tag(clause[i]->tag))
{
int fd = tcs->idxfd[clause[i]->tag];
lseek(fd, seek, SEEK_SET);
ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ);
if (tfe.tag_length >= (int)sizeof(str))
{
logf("Too long tag read!");
break ;
}
read(fd, str, tfe.tag_length);
/* Check if entry has been deleted. */
if (str[0] == '\0')
break;
}
if (!check_against_clause(seek, str, clause[i]))
return false;
}
}
return true;
}
bool tagcache_check_clauses(struct tagcache_search *tcs,
|