From df26e2e0c3e36ed02f87e249bee788d54886e9dd Mon Sep 17 00:00:00 2001

From: Carlos Garnacho <carlosg@gnome.org>

Date: Mon, 26 Dec 2016 21:49:44 +0100

Subject: [PATCH 1/8] libtracker-miner: Fully stop TrackerFileNotifier

 operations on errors

It oddly tried to just keep going, which may result in spurious deletes

or partial inserts. Sounds better to just drop the ball on the directory

at hand, it should be correctly indexed eventually.

---

 src/libtracker-miner/tracker-file-notifier.c | 22 +++++++++++++---------

 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/src/libtracker-miner/tracker-file-notifier.c b/src/libtracker-miner/tracker-file-notifier.c

index 4d3ec2467..5b1abe095 100644

--- a/src/libtracker-miner/tracker-file-notifier.c

+++ b/src/libtracker-miner/tracker-file-notifier.c

@@ -779,9 +779,11 @@ sparql_contents_query_cb (GObject      *object,

 	cursor = tracker_sparql_connection_query_finish (TRACKER_SPARQL_CONNECTION (object),

 	                                                 result, &error);

 	if (error) {

-		if (g_error_matches (error, G_IO_ERROR, G_IO_ERROR_CANCELLED))

-			goto out;

-		g_warning ("Could not query directory contents: %s\n", error->message);

+		if (!g_error_matches (error, G_IO_ERROR, G_IO_ERROR_CANCELLED)) {

+			g_warning ("Could not query directory contents: %s\n", error->message);

+			finish_current_directory (notifier, TRUE);

+		}

+		goto out;

 	}

 	notifier = user_data;

@@ -865,17 +867,19 @@ sparql_files_query_cb (GObject      *object,

 	GFile *directory;

 	guint flags;

+	notifier = data->notifier;

+	priv = notifier->priv;

+

 	cursor = tracker_sparql_connection_query_finish (TRACKER_SPARQL_CONNECTION (object),

 	                                                 result, &error);

 	if (error) {

-		if (g_error_matches (error, G_IO_ERROR, G_IO_ERROR_CANCELLED))

-			goto out;

-		g_warning ("Could not query indexed files: %s\n", error->message);

+		if (!g_error_matches (error, G_IO_ERROR, G_IO_ERROR_CANCELLED)) {

+			g_warning ("Could not query indexed files: %s\n", error->message);

+			finish_current_directory (notifier, TRUE);

+		}

+		goto out;

 	}

-	notifier = data->notifier;

-	priv = notifier->priv;

-

 	if (cursor) {

 		sparql_files_query_populate (notifier, cursor, TRUE);

 		g_object_unref (cursor);

-- 

2.14.2

From 1f5ac7a8c3db62d23c0148429905eeaf5c45b36d Mon Sep 17 00:00:00 2001

From: Carlos Garnacho <carlosg@gnome.org>

Date: Sun, 16 Apr 2017 11:58:00 -0300

Subject: [PATCH 2/8] libtracker-miner: Shuffle refcount handling when queueing

 back a file

The file might or might not be inserted to the queue, which meant that

the extra ref created outside the call might never dropped if the file

didn't end up inserted again. Fix this by doing the refcount increase

when actually inserting the file back in the queue.

Reported by Jose M. Arroyo <jose.m.arroyo.se@gmail.com>.

---

 src/libtracker-miner/tracker-miner-fs.c | 24 +++++++-----------------

 1 file changed, 7 insertions(+), 17 deletions(-)

diff --git a/src/libtracker-miner/tracker-miner-fs.c b/src/libtracker-miner/tracker-miner-fs.c

index 02b4a4755..15ac82f4d 100644

--- a/src/libtracker-miner/tracker-miner-fs.c

+++ b/src/libtracker-miner/tracker-miner-fs.c

@@ -2109,11 +2109,10 @@ should_wait (TrackerMinerFS *fs,

 }

 static gboolean

-item_reenqueue_full (TrackerMinerFS       *fs,

-                     TrackerPriorityQueue *item_queue,

-                     GFile                *queue_file,

-                     gpointer              queue_data,

-                     gint                  priority)

+item_enqueue_again (TrackerMinerFS       *fs,

+                    TrackerPriorityQueue *item_queue,

+                    GFile                *queue_file,

+                    gint                  priority)

 {

 	gint reentry_counter;

 	gchar *uri;

@@ -2126,7 +2125,7 @@ item_reenqueue_full (TrackerMinerFS       *fs,

 		g_object_set_qdata (G_OBJECT (queue_file),

 		                    fs->priv->quark_reentry_counter,

 		                    GINT_TO_POINTER (reentry_counter + 1));

-		tracker_priority_queue_add (item_queue, queue_data, priority);

+		tracker_priority_queue_add (item_queue, g_object_ref (queue_file), priority);

 		should_wait = TRUE;

 	} else {

@@ -2147,15 +2146,6 @@ item_reenqueue_full (TrackerMinerFS       *fs,

 	return should_wait;

 }

-static gboolean

-item_reenqueue (TrackerMinerFS       *fs,

-                TrackerPriorityQueue *item_queue,

-                GFile                *queue_file,

-                gint                  priority)

-{

-	return item_reenqueue_full (fs, item_queue, queue_file, queue_file, priority);

-}

-

 static QueueState

 item_queue_get_next_file (TrackerMinerFS  *fs,

                           GFile          **file,

@@ -2601,8 +2591,8 @@ item_queue_handlers_cb (gpointer user_data)

 			 * ensured, tasks are inserted at a higher priority so they

 			 * are processed promptly anyway.

 			 */

-			item_reenqueue (fs, item_queue, g_object_ref (parent), priority - 1);

-			item_reenqueue (fs, item_queue, g_object_ref (file), priority);

+			item_enqueue_again (fs, item_queue, parent, priority - 1);

+			item_enqueue_again (fs, item_queue, file, priority);

 			keep_processing = TRUE;

 		}

-- 

2.14.2

From cc917a0c029ebc46a1fbca5d73a9bd67b27e2e57 Mon Sep 17 00:00:00 2001

From: Carlos Garnacho <carlosg@gnome.org>

Date: Thu, 6 Jul 2017 11:19:17 +0200

Subject: [PATCH 3/8] libtracker-miner: Fix warnings if move ops happened

 during initial crawling

The checks to notify about indexing having finished on TrackerIndexingTree

roots were mistaking ItemMovedData* with GFile*, which lead to warnings.

This should be harmless, the signal might be possibly emitted before the

move op is dispatched, that's all.

---

 src/libtracker-miner/tracker-miner-fs.c | 11 ++++++++++-

 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/libtracker-miner/tracker-miner-fs.c b/src/libtracker-miner/tracker-miner-fs.c

index 15ac82f4d..baa2c9a7a 100644

--- a/src/libtracker-miner/tracker-miner-fs.c

+++ b/src/libtracker-miner/tracker-miner-fs.c

@@ -1110,6 +1110,15 @@ miner_resumed (TrackerMiner *miner)

 	}

 }

+static gboolean

+item_moved_data_has_prefix (gpointer data,

+			    gpointer user_data)

+{

+	ItemMovedData *moved_item = data;

+	GFile *prefix = user_data;

+

+	return g_file_has_prefix (moved_item->file, prefix);

+}

 static void

 miner_ignore_next_update (TrackerMiner *miner, const GStrv urls)

@@ -1165,7 +1174,7 @@ notify_roots_finished (TrackerMinerFS *fs,

 		    (tracker_priority_queue_find (fs->priv->items_created, NULL, (GEqualFunc) g_file_has_prefix, root) ||

 		     tracker_priority_queue_find (fs->priv->items_updated, NULL, (GEqualFunc) g_file_has_prefix, root) ||

 		     tracker_priority_queue_find (fs->priv->items_deleted, NULL, (GEqualFunc) g_file_has_prefix, root) ||

-		     tracker_priority_queue_find (fs->priv->items_moved, NULL, (GEqualFunc) g_file_has_prefix, root) ||

+		     tracker_priority_queue_find (fs->priv->items_moved, NULL, (GEqualFunc) item_moved_data_has_prefix, root) ||

 		     tracker_priority_queue_find (fs->priv->items_writeback, NULL, (GEqualFunc) g_file_has_prefix, root))) {

 			continue;

 		}

-- 

2.14.2

From 74ad92e34d8f58e9c8f015bf9f3173e3f738a23c Mon Sep 17 00:00:00 2001

From: Carlos Garnacho <carlosg@gnome.org>

Date: Sun, 9 Jul 2017 19:08:54 +0200

Subject: [PATCH 4/8] libtracker-miner: Ensure sparql buffer keeps flushing

The sparql buffer might get full again with new tasks before the

update_array operation for the current batch returned. In this case

nothing will kick the TrackerMinerFS again, nor the SPARQL buffer

from flushing again. Fix this by just flushing again, the miner

will follow as soon as the SPARQL buffer is below limits.

---

 src/libtracker-miner/tracker-sparql-buffer.c | 8 +++++++-

 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/libtracker-miner/tracker-sparql-buffer.c b/src/libtracker-miner/tracker-sparql-buffer.c

index 5d0be93a4..95f9c8ede 100644

--- a/src/libtracker-miner/tracker-sparql-buffer.c

+++ b/src/libtracker-miner/tracker-sparql-buffer.c

@@ -246,6 +246,7 @@ tracker_sparql_buffer_update_array_cb (GObject      *object,

                                        gpointer      user_data)

 {

 	TrackerSparqlBufferPrivate *priv;

+	TrackerSparqlBuffer *buffer;

 	GError *global_error = NULL;

 	GPtrArray *sparql_array_errors;

 	UpdateArrayData *update_data;

@@ -253,7 +254,8 @@ tracker_sparql_buffer_update_array_cb (GObject      *object,

 	/* Get arrays of errors and queries */

 	update_data = user_data;

-	priv = TRACKER_SPARQL_BUFFER (update_data->buffer)->priv;

+	buffer = TRACKER_SPARQL_BUFFER (update_data->buffer);

+	priv = buffer->priv;

 	priv->n_updates--;

 	g_debug ("(Sparql buffer) Finished array-update with %u tasks",

@@ -338,6 +340,10 @@ tracker_sparql_buffer_update_array_cb (GObject      *object,

 	if (global_error) {

 		g_error_free (global_error);

 	}

+

+	if (tracker_task_pool_limit_reached (TRACKER_TASK_POOL (buffer))) {

+		tracker_sparql_buffer_flush (buffer, "SPARQL buffer limit reached (after flush)");

+	}

 }

 static gchar *

-- 

2.14.2

From 534a1839a0f6e5bb71af7bc51b79f9d9e215266b Mon Sep 17 00:00:00 2001

From: Carlos Garnacho <carlosg@gnome.org>

Date: Mon, 25 Sep 2017 12:26:33 +0200

Subject: [PATCH 5/8] libtracker-miner: Check directory updates before dropping

 mtime info

file_notifier_traverse_tree() has the side effect of deleting mtime data,

as it's presumably not needed anymore, except here. Check whether the

directory was updated before notifying of the files inside, so we can

thoroughly check for deleted content.

https://bugzilla.gnome.org/show_bug.cgi?id=786132

---

 src/libtracker-miner/tracker-file-notifier.c | 7 +++++--

 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/libtracker-miner/tracker-file-notifier.c b/src/libtracker-miner/tracker-file-notifier.c

index 5b1abe095..433f79524 100644

--- a/src/libtracker-miner/tracker-file-notifier.c

+++ b/src/libtracker-miner/tracker-file-notifier.c

@@ -863,6 +863,7 @@ sparql_files_query_cb (GObject      *object,

 	TrackerFileNotifierPrivate *priv;

 	TrackerFileNotifier *notifier;

 	TrackerSparqlCursor *cursor;

+	gboolean directory_modified;

 	GError *error = NULL;

 	GFile *directory;

 	guint flags;

@@ -885,13 +886,15 @@ sparql_files_query_cb (GObject      *object,

 		g_object_unref (cursor);

 	}

-	file_notifier_traverse_tree (notifier, data->max_depth);

 	directory = priv->current_index_root->current_dir;

 	flags = priv->current_index_root->flags;

+	directory_modified = file_notifier_is_directory_modified (notifier, directory);

+

+	file_notifier_traverse_tree (notifier, data->max_depth);

 	if ((flags & TRACKER_DIRECTORY_FLAG_CHECK_DELETED) != 0 ||

 	    priv->current_index_root->current_dir_content_filtered ||

-	    file_notifier_is_directory_modified (notifier, directory)) {

+	    directory_modified) {

 		/* The directory has updated its mtime, this means something

 		 * was either added or removed in the mean time. Crawling

 		 * will always find all newly added files. But still, we

-- 

2.14.2

From c48c677e3a5ba0045868049e28a52c5e5e49b980 Mon Sep 17 00:00:00 2001

From: Carlos Garnacho <carlosg@gnome.org>

Date: Mon, 25 Sep 2017 12:12:47 +0200

Subject: [PATCH 6/8] libtracker-miner: Always fallback to URN query when

 queueing file

Tracker may end up with nfo:FileDataObject prior to handling monitor

events. Be it leftover data from previous bugs, explicit "tracker index"

calls, or data from some other application.

As we can't be really sure of the data consistence, always fallback to

a URN query so we don't break nie:url UNIQUE constraint (inverse functional

property in SPARQL parlance).

https://bugzilla.gnome.org/show_bug.cgi?id=786132

---

 src/libtracker-miner/tracker-miner-fs.c | 17 ++++++++---------

 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/src/libtracker-miner/tracker-miner-fs.c b/src/libtracker-miner/tracker-miner-fs.c

index baa2c9a7a..1a4af1c12 100644

--- a/src/libtracker-miner/tracker-miner-fs.c

+++ b/src/libtracker-miner/tracker-miner-fs.c

@@ -2821,12 +2821,11 @@ miner_fs_cache_file_urn (TrackerMinerFS *fs,

 static void

 miner_fs_queue_file (TrackerMinerFS       *fs,

                      TrackerPriorityQueue *item_queue,

-                     GFile                *file,

-                     gboolean              query_urn)

+                     GFile                *file)

 {

 	gint priority;

-	miner_fs_cache_file_urn (fs, file, query_urn);

+	miner_fs_cache_file_urn (fs, file, TRUE);

 	priority = miner_fs_get_queue_priority (fs, file);

 	tracker_priority_queue_add (item_queue, g_object_ref (file), priority);

 }

@@ -2981,7 +2980,7 @@ check_item_queues (TrackerMinerFS *fs,

 			 */

 			g_debug ("  Found matching unhandled CREATED event "

 			         "for source file, merging both events together");

-			miner_fs_queue_file (fs, fs->priv->items_created, other_file, FALSE);

+			miner_fs_queue_file (fs, fs->priv->items_created, other_file);

 			return FALSE;

 		}

@@ -3016,7 +3015,7 @@ file_notifier_file_created (TrackerFileNotifier  *notifier,

 	TrackerMinerFS *fs = user_data;

 	if (check_item_queues (fs, QUEUE_CREATED, file, NULL)) {

-		miner_fs_queue_file (fs, fs->priv->items_created, file, FALSE);

+		miner_fs_queue_file (fs, fs->priv->items_created, file);

 		item_queue_handlers_set_up (fs);

 	}

 }

@@ -3039,7 +3038,7 @@ file_notifier_file_deleted (TrackerFileNotifier  *notifier,

 	}

 	if (check_item_queues (fs, QUEUE_DELETED, file, NULL)) {

-		miner_fs_queue_file (fs, fs->priv->items_deleted, file, FALSE);

+		miner_fs_queue_file (fs, fs->priv->items_deleted, file);

 		item_queue_handlers_set_up (fs);

 	}

 }

@@ -3070,7 +3069,7 @@ file_notifier_file_updated (TrackerFileNotifier  *notifier,

 			                    GINT_TO_POINTER (TRUE));

 		}

-		miner_fs_queue_file (fs, fs->priv->items_updated, file, TRUE);

+		miner_fs_queue_file (fs, fs->priv->items_updated, file);

 		item_queue_handlers_set_up (fs);

 	}

 }

@@ -3416,7 +3415,7 @@ tracker_miner_fs_directory_remove_full (TrackerMinerFS *fs,

 			 * to preserve remove_full() semantics.

 			 */

 			trace_eq_push_tail ("DELETED", file, "on remove full");

-			miner_fs_queue_file (fs, fs->priv->items_deleted, file, FALSE);

+			miner_fs_queue_file (fs, fs->priv->items_deleted, file);

 			item_queue_handlers_set_up (fs);

 		}

@@ -3460,7 +3459,7 @@ check_file_parents (TrackerMinerFS *fs,

 	for (p = parents; p; p = p->next) {

 		trace_eq_push_tail ("UPDATED", p->data, "checking file parents");

-		miner_fs_queue_file (fs, fs->priv->items_updated, p->data, TRUE);

+		miner_fs_queue_file (fs, fs->priv->items_updated, p->data);

 		g_object_unref (p->data);

 	}

-- 

2.14.2

From 9df5d3cae1c79bdb3c5f930bf665fe5d1f442547 Mon Sep 17 00:00:00 2001

From: Carlos Garnacho <carlosg@gnome.org>

Date: Sun, 24 Sep 2017 12:04:22 +0200

Subject: [PATCH 7/8] tracker-miner-fs: Explicitly ignore non-native mounts

tracker-miner-fs does rely in a few places to index local content

exclusively. Make it explicitly ignore non-native mounts, they won't

be indexed anyway, but we can spare trying to create the tracker:Volume

and mount root's nfo:FileDataObject for these.

Fixes nie:url constraint errors and other warnings when trying to deal

with those.

https://bugzilla.gnome.org/show_bug.cgi?id=786132

---

 src/miners/fs/tracker-storage.c | 10 +++++++++-

 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/miners/fs/tracker-storage.c b/src/miners/fs/tracker-storage.c

index 9ec59dbfd..6114a6fd6 100644

--- a/src/miners/fs/tracker-storage.c

+++ b/src/miners/fs/tracker-storage.c

@@ -550,8 +550,16 @@ mount_add (TrackerStorage *storage,

 	/* Get root path of the mount */

 	root = g_mount_get_root (mount);

-	mount_path = g_file_get_path (root);

+	if (!g_file_is_native (root)) {

+		gchar *uri = g_file_get_uri (root);

+		g_debug ("Ignoring mount '%s', URI '%s' is not native",

+		         mount_name, uri);

+		g_free (uri);

+		return;

+	}

+

+	mount_path = g_file_get_path (root);

 	g_debug ("Found '%s' mounted on path '%s'",

 	         mount_name,

 	         mount_path);

-- 

2.14.2

From 907a867cd72827d5b24d6d8474d9e76d6ff095aa Mon Sep 17 00:00:00 2001

From: Carlos Garnacho <carlosg@gnome.org>

Date: Sun, 10 Sep 2017 17:01:26 +0200

Subject: [PATCH 8/8] applications: Ignore broken symlinks

We still do need to query mtime from the symlink itself in order to keep

filesystem accounting happy, so wait for ENOENT while doing

g_key_file_load_from_file() in order to figure out we are dealing with

a broken link (or a no longer existing file, which is also possible). This

avoids spewing any warnings with those.

https://bugzilla.gnome.org/show_bug.cgi?id=786132

---

 src/miners/apps/tracker-miner-applications.c | 34 ++++++++++++++++------------

 1 file changed, 20 insertions(+), 14 deletions(-)

diff --git a/src/miners/apps/tracker-miner-applications.c b/src/miners/apps/tracker-miner-applications.c

index 749d3e4d2..a2d1f78d8 100644

--- a/src/miners/apps/tracker-miner-applications.c

+++ b/src/miners/apps/tracker-miner-applications.c

@@ -430,8 +430,7 @@ get_desktop_key_file (GFile   *file,

 	key_file = g_key_file_new ();

 	*type = NULL;

-	if (!g_key_file_load_from_file (key_file, path, G_KEY_FILE_NONE, NULL)) {

-		g_set_error (error, miner_applications_error_quark, 0, "Couldn't load desktop file:'%s'", path);

+	if (!g_key_file_load_from_file (key_file, path, G_KEY_FILE_NONE, error)) {

 		g_key_file_free (key_file);

 		g_free (path);

 		return NULL;

@@ -943,6 +942,7 @@ process_file_cb (GObject      *object,

 	ProcessApplicationData *data;

 	GFileInfo *file_info;

 	GError *error = NULL;

+	GFileType file_type;

 	GFile *file;

 	data = user_data;

@@ -956,21 +956,27 @@ process_file_cb (GObject      *object,

 		return;

 	}

-	if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_DIRECTORY) {

+	file_type = g_file_info_get_file_type (file_info);

+

+	if (file_type == G_FILE_TYPE_DIRECTORY) {

 		process_directory (data, file_info, &error);

-	} else {

+	} else if (file_type == G_FILE_TYPE_REGULAR ||

+	           file_type == G_FILE_TYPE_SYMBOLIC_LINK) {

 		data->key_file = get_desktop_key_file (file, &data->type, &error);

 		if (!data->key_file) {

-			gchar *uri;

-

-			uri = g_file_get_uri (file);

-			g_warning ("Couldn't properly parse desktop file '%s': '%s'",

-			           uri,

-			           error ? error->message : "unknown error");

-			g_free (uri);

-			g_clear_error (&error);

-

-			error = g_error_new_literal (miner_applications_error_quark, 0, "File is not a key file");

+			/* Ignore broken symlinks */

+			if (!g_error_matches (error, G_FILE_ERROR, G_FILE_ERROR_NOENT)) {

+				gchar *uri;

+

+				uri = g_file_get_uri (file);

+				g_warning ("Couldn't properly parse desktop file '%s': '%s'",

+				           uri,

+				           error ? error->message : "unknown error");

+				g_free (uri);

+				g_clear_error (&error);

+

+				error = g_error_new_literal (miner_applications_error_quark, 0, "File is not a key file");

+			}

 		} else if (g_key_file_get_boolean (data->key_file, GROUP_DESKTOP_ENTRY, "Hidden", NULL)) {

 			error = g_error_new_literal (miner_applications_error_quark, 0, "Desktop file is 'hidden', not gathering metadata for it");

 		} else {

-- 

2.14.2