From df26e2e0c3e36ed02f87e249bee788d54886e9dd Mon Sep 17 00:00:00 2001
From: Carlos Garnacho <carlosg@gnome.org>
Date: Mon, 26 Dec 2016 21:49:44 +0100
Subject: [PATCH 1/8] libtracker-miner: Fully stop TrackerFileNotifier
operations on errors
It oddly tried to just keep going, which may result in spurious deletes
or partial inserts. Sounds better to just drop the ball on the directory
at hand, it should be correctly indexed eventually.
---
src/libtracker-miner/tracker-file-notifier.c | 22 +++++++++++++---------
1 file changed, 13 insertions(+), 9 deletions(-)
diff --git a/src/libtracker-miner/tracker-file-notifier.c b/src/libtracker-miner/tracker-file-notifier.c
index 4d3ec2467..5b1abe095 100644
--- a/src/libtracker-miner/tracker-file-notifier.c
+++ b/src/libtracker-miner/tracker-file-notifier.c
@@ -779,9 +779,11 @@ sparql_contents_query_cb (GObject *object,
cursor = tracker_sparql_connection_query_finish (TRACKER_SPARQL_CONNECTION (object),
result, &error);
if (error) {
- if (g_error_matches (error, G_IO_ERROR, G_IO_ERROR_CANCELLED))
- goto out;
- g_warning ("Could not query directory contents: %s\n", error->message);
+ if (!g_error_matches (error, G_IO_ERROR, G_IO_ERROR_CANCELLED)) {
+ g_warning ("Could not query directory contents: %s\n", error->message);
+ finish_current_directory (notifier, TRUE);
+ }
+ goto out;
}
notifier = user_data;
@@ -865,17 +867,19 @@ sparql_files_query_cb (GObject *object,
GFile *directory;
guint flags;
+ notifier = data->notifier;
+ priv = notifier->priv;
+
cursor = tracker_sparql_connection_query_finish (TRACKER_SPARQL_CONNECTION (object),
result, &error);
if (error) {
- if (g_error_matches (error, G_IO_ERROR, G_IO_ERROR_CANCELLED))
- goto out;
- g_warning ("Could not query indexed files: %s\n", error->message);
+ if (!g_error_matches (error, G_IO_ERROR, G_IO_ERROR_CANCELLED)) {
+ g_warning ("Could not query indexed files: %s\n", error->message);
+ finish_current_directory (notifier, TRUE);
+ }
+ goto out;
}
- notifier = data->notifier;
- priv = notifier->priv;
-
if (cursor) {
sparql_files_query_populate (notifier, cursor, TRUE);
g_object_unref (cursor);
--
2.14.2
From 1f5ac7a8c3db62d23c0148429905eeaf5c45b36d Mon Sep 17 00:00:00 2001
From: Carlos Garnacho <carlosg@gnome.org>
Date: Sun, 16 Apr 2017 11:58:00 -0300
Subject: [PATCH 2/8] libtracker-miner: Shuffle refcount handling when queueing
back a file
The file might or might not be inserted to the queue, which meant that
the extra ref created outside the call might never dropped if the file
didn't end up inserted again. Fix this by doing the refcount increase
when actually inserting the file back in the queue.
Reported by Jose M. Arroyo <jose.m.arroyo.se@gmail.com>.
---
src/libtracker-miner/tracker-miner-fs.c | 24 +++++++-----------------
1 file changed, 7 insertions(+), 17 deletions(-)
diff --git a/src/libtracker-miner/tracker-miner-fs.c b/src/libtracker-miner/tracker-miner-fs.c
index 02b4a4755..15ac82f4d 100644
--- a/src/libtracker-miner/tracker-miner-fs.c
+++ b/src/libtracker-miner/tracker-miner-fs.c
@@ -2109,11 +2109,10 @@ should_wait (TrackerMinerFS *fs,
}
static gboolean
-item_reenqueue_full (TrackerMinerFS *fs,
- TrackerPriorityQueue *item_queue,
- GFile *queue_file,
- gpointer queue_data,
- gint priority)
+item_enqueue_again (TrackerMinerFS *fs,
+ TrackerPriorityQueue *item_queue,
+ GFile *queue_file,
+ gint priority)
{
gint reentry_counter;
gchar *uri;
@@ -2126,7 +2125,7 @@ item_reenqueue_full (TrackerMinerFS *fs,
g_object_set_qdata (G_OBJECT (queue_file),
fs->priv->quark_reentry_counter,
GINT_TO_POINTER (reentry_counter + 1));
- tracker_priority_queue_add (item_queue, queue_data, priority);
+ tracker_priority_queue_add (item_queue, g_object_ref (queue_file), priority);
should_wait = TRUE;
} else {
@@ -2147,15 +2146,6 @@ item_reenqueue_full (TrackerMinerFS *fs,
return should_wait;
}
-static gboolean
-item_reenqueue (TrackerMinerFS *fs,
- TrackerPriorityQueue *item_queue,
- GFile *queue_file,
- gint priority)
-{
- return item_reenqueue_full (fs, item_queue, queue_file, queue_file, priority);
-}
-
static QueueState
item_queue_get_next_file (TrackerMinerFS *fs,
GFile **file,
@@ -2601,8 +2591,8 @@ item_queue_handlers_cb (gpointer user_data)
* ensured, tasks are inserted at a higher priority so they
* are processed promptly anyway.
*/
- item_reenqueue (fs, item_queue, g_object_ref (parent), priority - 1);
- item_reenqueue (fs, item_queue, g_object_ref (file), priority);
+ item_enqueue_again (fs, item_queue, parent, priority - 1);
+ item_enqueue_again (fs, item_queue, file, priority);
keep_processing = TRUE;
}
--
2.14.2
From cc917a0c029ebc46a1fbca5d73a9bd67b27e2e57 Mon Sep 17 00:00:00 2001
From: Carlos Garnacho <carlosg@gnome.org>
Date: Thu, 6 Jul 2017 11:19:17 +0200
Subject: [PATCH 3/8] libtracker-miner: Fix warnings if move ops happened
during initial crawling
The checks to notify about indexing having finished on TrackerIndexingTree
roots were mistaking ItemMovedData* with GFile*, which lead to warnings.
This should be harmless, the signal might be possibly emitted before the
move op is dispatched, that's all.
---
src/libtracker-miner/tracker-miner-fs.c | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
diff --git a/src/libtracker-miner/tracker-miner-fs.c b/src/libtracker-miner/tracker-miner-fs.c
index 15ac82f4d..baa2c9a7a 100644
--- a/src/libtracker-miner/tracker-miner-fs.c
+++ b/src/libtracker-miner/tracker-miner-fs.c
@@ -1110,6 +1110,15 @@ miner_resumed (TrackerMiner *miner)
}
}
+static gboolean
+item_moved_data_has_prefix (gpointer data,
+ gpointer user_data)
+{
+ ItemMovedData *moved_item = data;
+ GFile *prefix = user_data;
+
+ return g_file_has_prefix (moved_item->file, prefix);
+}
static void
miner_ignore_next_update (TrackerMiner *miner, const GStrv urls)
@@ -1165,7 +1174,7 @@ notify_roots_finished (TrackerMinerFS *fs,
(tracker_priority_queue_find (fs->priv->items_created, NULL, (GEqualFunc) g_file_has_prefix, root) ||
tracker_priority_queue_find (fs->priv->items_updated, NULL, (GEqualFunc) g_file_has_prefix, root) ||
tracker_priority_queue_find (fs->priv->items_deleted, NULL, (GEqualFunc) g_file_has_prefix, root) ||
- tracker_priority_queue_find (fs->priv->items_moved, NULL, (GEqualFunc) g_file_has_prefix, root) ||
+ tracker_priority_queue_find (fs->priv->items_moved, NULL, (GEqualFunc) item_moved_data_has_prefix, root) ||
tracker_priority_queue_find (fs->priv->items_writeback, NULL, (GEqualFunc) g_file_has_prefix, root))) {
continue;
}
--
2.14.2
From 74ad92e34d8f58e9c8f015bf9f3173e3f738a23c Mon Sep 17 00:00:00 2001
From: Carlos Garnacho <carlosg@gnome.org>
Date: Sun, 9 Jul 2017 19:08:54 +0200
Subject: [PATCH 4/8] libtracker-miner: Ensure sparql buffer keeps flushing
The sparql buffer might get full again with new tasks before the
update_array operation for the current batch returned. In this case
nothing will kick the TrackerMinerFS again, nor the SPARQL buffer
from flushing again. Fix this by just flushing again, the miner
will follow as soon as the SPARQL buffer is below limits.
---
src/libtracker-miner/tracker-sparql-buffer.c | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/src/libtracker-miner/tracker-sparql-buffer.c b/src/libtracker-miner/tracker-sparql-buffer.c
index 5d0be93a4..95f9c8ede 100644
--- a/src/libtracker-miner/tracker-sparql-buffer.c
+++ b/src/libtracker-miner/tracker-sparql-buffer.c
@@ -246,6 +246,7 @@ tracker_sparql_buffer_update_array_cb (GObject *object,
gpointer user_data)
{
TrackerSparqlBufferPrivate *priv;
+ TrackerSparqlBuffer *buffer;
GError *global_error = NULL;
GPtrArray *sparql_array_errors;
UpdateArrayData *update_data;
@@ -253,7 +254,8 @@ tracker_sparql_buffer_update_array_cb (GObject *object,
/* Get arrays of errors and queries */
update_data = user_data;
- priv = TRACKER_SPARQL_BUFFER (update_data->buffer)->priv;
+ buffer = TRACKER_SPARQL_BUFFER (update_data->buffer);
+ priv = buffer->priv;
priv->n_updates--;
g_debug ("(Sparql buffer) Finished array-update with %u tasks",
@@ -338,6 +340,10 @@ tracker_sparql_buffer_update_array_cb (GObject *object,
if (global_error) {
g_error_free (global_error);
}
+
+ if (tracker_task_pool_limit_reached (TRACKER_TASK_POOL (buffer))) {
+ tracker_sparql_buffer_flush (buffer, "SPARQL buffer limit reached (after flush)");
+ }
}
static gchar *
--
2.14.2
From 534a1839a0f6e5bb71af7bc51b79f9d9e215266b Mon Sep 17 00:00:00 2001
From: Carlos Garnacho <carlosg@gnome.org>
Date: Mon, 25 Sep 2017 12:26:33 +0200
Subject: [PATCH 5/8] libtracker-miner: Check directory updates before dropping
mtime info
file_notifier_traverse_tree() has the side effect of deleting mtime data,
as it's presumably not needed anymore, except here. Check whether the
directory was updated before notifying of the files inside, so we can
thoroughly check for deleted content.
https://bugzilla.gnome.org/show_bug.cgi?id=786132
---
src/libtracker-miner/tracker-file-notifier.c | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/src/libtracker-miner/tracker-file-notifier.c b/src/libtracker-miner/tracker-file-notifier.c
index 5b1abe095..433f79524 100644
--- a/src/libtracker-miner/tracker-file-notifier.c
+++ b/src/libtracker-miner/tracker-file-notifier.c
@@ -863,6 +863,7 @@ sparql_files_query_cb (GObject *object,
TrackerFileNotifierPrivate *priv;
TrackerFileNotifier *notifier;
TrackerSparqlCursor *cursor;
+ gboolean directory_modified;
GError *error = NULL;
GFile *directory;
guint flags;
@@ -885,13 +886,15 @@ sparql_files_query_cb (GObject *object,
g_object_unref (cursor);
}
- file_notifier_traverse_tree (notifier, data->max_depth);
directory = priv->current_index_root->current_dir;
flags = priv->current_index_root->flags;
+ directory_modified = file_notifier_is_directory_modified (notifier, directory);
+
+ file_notifier_traverse_tree (notifier, data->max_depth);
if ((flags & TRACKER_DIRECTORY_FLAG_CHECK_DELETED) != 0 ||
priv->current_index_root->current_dir_content_filtered ||
- file_notifier_is_directory_modified (notifier, directory)) {
+ directory_modified) {
/* The directory has updated its mtime, this means something
* was either added or removed in the mean time. Crawling
* will always find all newly added files. But still, we
--
2.14.2
From c48c677e3a5ba0045868049e28a52c5e5e49b980 Mon Sep 17 00:00:00 2001
From: Carlos Garnacho <carlosg@gnome.org>
Date: Mon, 25 Sep 2017 12:12:47 +0200
Subject: [PATCH 6/8] libtracker-miner: Always fallback to URN query when
queueing file
Tracker may end up with nfo:FileDataObject prior to handling monitor
events. Be it leftover data from previous bugs, explicit "tracker index"
calls, or data from some other application.
As we can't be really sure of the data consistence, always fallback to
a URN query so we don't break nie:url UNIQUE constraint (inverse functional
property in SPARQL parlance).
https://bugzilla.gnome.org/show_bug.cgi?id=786132
---
src/libtracker-miner/tracker-miner-fs.c | 17 ++++++++---------
1 file changed, 8 insertions(+), 9 deletions(-)
diff --git a/src/libtracker-miner/tracker-miner-fs.c b/src/libtracker-miner/tracker-miner-fs.c
index baa2c9a7a..1a4af1c12 100644
--- a/src/libtracker-miner/tracker-miner-fs.c
+++ b/src/libtracker-miner/tracker-miner-fs.c
@@ -2821,12 +2821,11 @@ miner_fs_cache_file_urn (TrackerMinerFS *fs,
static void
miner_fs_queue_file (TrackerMinerFS *fs,
TrackerPriorityQueue *item_queue,
- GFile *file,
- gboolean query_urn)
+ GFile *file)
{
gint priority;
- miner_fs_cache_file_urn (fs, file, query_urn);
+ miner_fs_cache_file_urn (fs, file, TRUE);
priority = miner_fs_get_queue_priority (fs, file);
tracker_priority_queue_add (item_queue, g_object_ref (file), priority);
}
@@ -2981,7 +2980,7 @@ check_item_queues (TrackerMinerFS *fs,
*/
g_debug (" Found matching unhandled CREATED event "
"for source file, merging both events together");
- miner_fs_queue_file (fs, fs->priv->items_created, other_file, FALSE);
+ miner_fs_queue_file (fs, fs->priv->items_created, other_file);
return FALSE;
}
@@ -3016,7 +3015,7 @@ file_notifier_file_created (TrackerFileNotifier *notifier,
TrackerMinerFS *fs = user_data;
if (check_item_queues (fs, QUEUE_CREATED, file, NULL)) {
- miner_fs_queue_file (fs, fs->priv->items_created, file, FALSE);
+ miner_fs_queue_file (fs, fs->priv->items_created, file);
item_queue_handlers_set_up (fs);
}
}
@@ -3039,7 +3038,7 @@ file_notifier_file_deleted (TrackerFileNotifier *notifier,
}
if (check_item_queues (fs, QUEUE_DELETED, file, NULL)) {
- miner_fs_queue_file (fs, fs->priv->items_deleted, file, FALSE);
+ miner_fs_queue_file (fs, fs->priv->items_deleted, file);
item_queue_handlers_set_up (fs);
}
}
@@ -3070,7 +3069,7 @@ file_notifier_file_updated (TrackerFileNotifier *notifier,
GINT_TO_POINTER (TRUE));
}
- miner_fs_queue_file (fs, fs->priv->items_updated, file, TRUE);
+ miner_fs_queue_file (fs, fs->priv->items_updated, file);
item_queue_handlers_set_up (fs);
}
}
@@ -3416,7 +3415,7 @@ tracker_miner_fs_directory_remove_full (TrackerMinerFS *fs,
* to preserve remove_full() semantics.
*/
trace_eq_push_tail ("DELETED", file, "on remove full");
- miner_fs_queue_file (fs, fs->priv->items_deleted, file, FALSE);
+ miner_fs_queue_file (fs, fs->priv->items_deleted, file);
item_queue_handlers_set_up (fs);
}
@@ -3460,7 +3459,7 @@ check_file_parents (TrackerMinerFS *fs,
for (p = parents; p; p = p->next) {
trace_eq_push_tail ("UPDATED", p->data, "checking file parents");
- miner_fs_queue_file (fs, fs->priv->items_updated, p->data, TRUE);
+ miner_fs_queue_file (fs, fs->priv->items_updated, p->data);
g_object_unref (p->data);
}
--
2.14.2
From 9df5d3cae1c79bdb3c5f930bf665fe5d1f442547 Mon Sep 17 00:00:00 2001
From: Carlos Garnacho <carlosg@gnome.org>
Date: Sun, 24 Sep 2017 12:04:22 +0200
Subject: [PATCH 7/8] tracker-miner-fs: Explicitly ignore non-native mounts
tracker-miner-fs does rely in a few places to index local content
exclusively. Make it explicitly ignore non-native mounts, they won't
be indexed anyway, but we can spare trying to create the tracker:Volume
and mount root's nfo:FileDataObject for these.
Fixes nie:url constraint errors and other warnings when trying to deal
with those.
https://bugzilla.gnome.org/show_bug.cgi?id=786132
---
src/miners/fs/tracker-storage.c | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
diff --git a/src/miners/fs/tracker-storage.c b/src/miners/fs/tracker-storage.c
index 9ec59dbfd..6114a6fd6 100644
--- a/src/miners/fs/tracker-storage.c
+++ b/src/miners/fs/tracker-storage.c
@@ -550,8 +550,16 @@ mount_add (TrackerStorage *storage,
/* Get root path of the mount */
root = g_mount_get_root (mount);
- mount_path = g_file_get_path (root);
+ if (!g_file_is_native (root)) {
+ gchar *uri = g_file_get_uri (root);
+ g_debug ("Ignoring mount '%s', URI '%s' is not native",
+ mount_name, uri);
+ g_free (uri);
+ return;
+ }
+
+ mount_path = g_file_get_path (root);
g_debug ("Found '%s' mounted on path '%s'",
mount_name,
mount_path);
--
2.14.2
From 907a867cd72827d5b24d6d8474d9e76d6ff095aa Mon Sep 17 00:00:00 2001
From: Carlos Garnacho <carlosg@gnome.org>
Date: Sun, 10 Sep 2017 17:01:26 +0200
Subject: [PATCH 8/8] applications: Ignore broken symlinks
We still do need to query mtime from the symlink itself in order to keep
filesystem accounting happy, so wait for ENOENT while doing
g_key_file_load_from_file() in order to figure out we are dealing with
a broken link (or a no longer existing file, which is also possible). This
avoids spewing any warnings with those.
https://bugzilla.gnome.org/show_bug.cgi?id=786132
---
src/miners/apps/tracker-miner-applications.c | 34 ++++++++++++++++------------
1 file changed, 20 insertions(+), 14 deletions(-)
diff --git a/src/miners/apps/tracker-miner-applications.c b/src/miners/apps/tracker-miner-applications.c
index 749d3e4d2..a2d1f78d8 100644
--- a/src/miners/apps/tracker-miner-applications.c
+++ b/src/miners/apps/tracker-miner-applications.c
@@ -430,8 +430,7 @@ get_desktop_key_file (GFile *file,
key_file = g_key_file_new ();
*type = NULL;
- if (!g_key_file_load_from_file (key_file, path, G_KEY_FILE_NONE, NULL)) {
- g_set_error (error, miner_applications_error_quark, 0, "Couldn't load desktop file:'%s'", path);
+ if (!g_key_file_load_from_file (key_file, path, G_KEY_FILE_NONE, error)) {
g_key_file_free (key_file);
g_free (path);
return NULL;
@@ -943,6 +942,7 @@ process_file_cb (GObject *object,
ProcessApplicationData *data;
GFileInfo *file_info;
GError *error = NULL;
+ GFileType file_type;
GFile *file;
data = user_data;
@@ -956,21 +956,27 @@ process_file_cb (GObject *object,
return;
}
- if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_DIRECTORY) {
+ file_type = g_file_info_get_file_type (file_info);
+
+ if (file_type == G_FILE_TYPE_DIRECTORY) {
process_directory (data, file_info, &error);
- } else {
+ } else if (file_type == G_FILE_TYPE_REGULAR ||
+ file_type == G_FILE_TYPE_SYMBOLIC_LINK) {
data->key_file = get_desktop_key_file (file, &data->type, &error);
if (!data->key_file) {
- gchar *uri;
-
- uri = g_file_get_uri (file);
- g_warning ("Couldn't properly parse desktop file '%s': '%s'",
- uri,
- error ? error->message : "unknown error");
- g_free (uri);
- g_clear_error (&error);
-
- error = g_error_new_literal (miner_applications_error_quark, 0, "File is not a key file");
+ /* Ignore broken symlinks */
+ if (!g_error_matches (error, G_FILE_ERROR, G_FILE_ERROR_NOENT)) {
+ gchar *uri;
+
+ uri = g_file_get_uri (file);
+ g_warning ("Couldn't properly parse desktop file '%s': '%s'",
+ uri,
+ error ? error->message : "unknown error");
+ g_free (uri);
+ g_clear_error (&error);
+
+ error = g_error_new_literal (miner_applications_error_quark, 0, "File is not a key file");
+ }
} else if (g_key_file_get_boolean (data->key_file, GROUP_DESKTOP_ENTRY, "Hidden", NULL)) {
error = g_error_new_literal (miner_applications_error_quark, 0, "Desktop file is 'hidden', not gathering metadata for it");
} else {
--
2.14.2