From a1913855ba9665138c98e7310842d79c4f49871d Mon Sep 17 00:00:00 2001
From: Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>
Date: Thu, 27 May 2021 12:30:57 -0400
Subject: [PATCH 1/6] drm/panfrost: Add cycle counter job requirement

Extend the Panfrost UABI with a new job requirement for cycle counters
(and GPU timestamps, by extension). This requirement is used in
userspace to implement ARB_shader_clock, an OpenGL extension reporting
the GPU cycle count within a shader. The same mechanism will be required
to implement timestamp queries as a "write value - timestamp" job.

We cannot enable cycle counters unconditionally, as enabling them
increases GPU power consumption. They should be left off unless actually
required by the application for profiling purposes.

Signed-off-by: Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>
---
 include/uapi/drm/panfrost_drm.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/include/uapi/drm/panfrost_drm.h b/include/uapi/drm/panfrost_drm.h
index ec19db1ee..27e6cb941 100644
--- a/include/uapi/drm/panfrost_drm.h
+++ b/include/uapi/drm/panfrost_drm.h
@@ -39,7 +39,8 @@ extern "C" {
 #define DRM_IOCTL_PANFROST_PERFCNT_ENABLE	DRM_IOW(DRM_COMMAND_BASE + DRM_PANFROST_PERFCNT_ENABLE, struct drm_panfrost_perfcnt_enable)
 #define DRM_IOCTL_PANFROST_PERFCNT_DUMP		DRM_IOW(DRM_COMMAND_BASE + DRM_PANFROST_PERFCNT_DUMP, struct drm_panfrost_perfcnt_dump)
 
-#define PANFROST_JD_REQ_FS (1 << 0)
+#define PANFROST_JD_REQ_FS			(1 << 0)
+#define PANFROST_JD_REQ_PERMON			(1 << 1)
 /**
  * struct drm_panfrost_submit - ioctl argument for submitting commands to the 3D
  * engine.
-- 
2.30.2

From 602e24387356947481fffa7c43d96df122702cfe Mon Sep 17 00:00:00 2001
From: Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>
Date: Thu, 27 May 2021 12:31:08 -0400
Subject: [PATCH 2/6] drm/panfrost: Add cycle counter registers

Add additional values CYCLE_COUNTER_{START,STOP} of GPU_COMMAND required
to enable and disable the cycle counter. Similarly add the GPU timestamp
and cycle count registers that this command enables/disables, and add
the GPU_STATUS bit indicating whether they are indeed enabled. Values
from mali_kbase.

Signed-off-by: Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>
---
 drivers/gpu/drm/panfrost/panfrost_regs.h | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/drivers/gpu/drm/panfrost/panfrost_regs.h b/drivers/gpu/drm/panfrost/panfrost_regs.h
index eddaa62ad..674dc9b79 100644
--- a/drivers/gpu/drm/panfrost/panfrost_regs.h
+++ b/drivers/gpu/drm/panfrost/panfrost_regs.h
@@ -46,10 +46,13 @@
 #define   GPU_CMD_SOFT_RESET		0x01
 #define   GPU_CMD_PERFCNT_CLEAR		0x03
 #define   GPU_CMD_PERFCNT_SAMPLE	0x04
+#define   GPU_CMD_CYCLE_COUNT_START	0x05
+#define   GPU_CMD_CYCLE_COUNT_STOP	0x06
 #define   GPU_CMD_CLEAN_CACHES		0x07
 #define   GPU_CMD_CLEAN_INV_CACHES	0x08
 #define GPU_STATUS			0x34
 #define   GPU_STATUS_PRFCNT_ACTIVE	BIT(2)
+#define   GPU_STATUS_CYCLE_COUNT_ACTIVE	BIT(6)
 #define GPU_LATEST_FLUSH_ID		0x38
 #define GPU_PWR_KEY			0x50	/* (WO) Power manager key register */
 #define  GPU_PWR_KEY_UNLOCK		0x2968A819
@@ -72,6 +75,10 @@
 #define GPU_PRFCNT_SHADER_EN		0x70
 #define GPU_PRFCNT_TILER_EN		0x74
 #define GPU_PRFCNT_MMU_L2_EN		0x7c
+#define GPU_CYCLE_COUNT_LO		0x90
+#define GPU_CYCLE_COUNT_HI		0x94
+#define GPU_TIMESTAMP_LO		0x98
+#define GPU_TIMESTAMP_HI		0x9C
 
 #define GPU_THREAD_MAX_THREADS		0x0A0	/* (RO) Maximum number of threads per core */
 #define GPU_THREAD_MAX_WORKGROUP_SIZE	0x0A4	/* (RO) Maximum workgroup size */
-- 
2.30.2

From 5905d7636739e6c842945dcefb3adbd96beda819 Mon Sep 17 00:00:00 2001
From: Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>
Date: Thu, 27 May 2021 12:31:11 -0400
Subject: [PATCH 3/6] drm/panfrost: Add permon acquire/release helpers

Wrap the underlying CYCLE_COUNT_START/STOP commands in a safe interface
that ensures the commands are only issued where required by guarding
behind an atomic counter. In particular, we need to be careful about
races between multiple in-flight jobs, where only some require cycle
counts.

Signed-off-by: Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>
---
 drivers/gpu/drm/panfrost/panfrost_device.h |  3 +++
 drivers/gpu/drm/panfrost/panfrost_gpu.c    | 20 ++++++++++++++++++++
 drivers/gpu/drm/panfrost/panfrost_gpu.h    |  3 +++
 3 files changed, 26 insertions(+)

diff --git a/drivers/gpu/drm/panfrost/panfrost_device.h b/drivers/gpu/drm/panfrost/panfrost_device.h
index 597cf1459..8a89aa274 100644
--- a/drivers/gpu/drm/panfrost/panfrost_device.h
+++ b/drivers/gpu/drm/panfrost/panfrost_device.h
@@ -117,6 +117,9 @@ struct panfrost_device {
 	struct shrinker shrinker;
 
 	struct panfrost_devfreq pfdevfreq;
+
+	/* Number of active jobs requiring performance monitoring */
+	atomic_t permon_pending;
 };
 
 struct panfrost_mmu {
diff --git a/drivers/gpu/drm/panfrost/panfrost_gpu.c b/drivers/gpu/drm/panfrost/panfrost_gpu.c
index 2aae636f1..acacceb15 100644
--- a/drivers/gpu/drm/panfrost/panfrost_gpu.c
+++ b/drivers/gpu/drm/panfrost/panfrost_gpu.c
@@ -399,3 +399,23 @@ u32 panfrost_gpu_get_latest_flush_id(struct panfrost_device *pfdev)
 
 	return 0;
 }
+
+void panfrost_acquire_permon(struct panfrost_device *pfdev)
+{
+	/* If another in-flight job enabled permon, we don't have to */
+	if (atomic_inc_return(&pfdev->permon_pending) > 1)
+		return;
+
+	/* Otherwise, we're the first user */
+	gpu_write(pfdev, GPU_CMD, GPU_CMD_CYCLE_COUNT_START);
+}
+
+void panfrost_release_permon(struct panfrost_device *pfdev)
+{
+	/* If another in-flight job needs permon, keep it active */
+	if (atomic_dec_return(&pfdev->permon_pending) > 0)
+		return;
+
+	/* Otherwise, we're the last user */
+	gpu_write(pfdev, GPU_CMD, GPU_CMD_CYCLE_COUNT_STOP);
+}
diff --git a/drivers/gpu/drm/panfrost/panfrost_gpu.h b/drivers/gpu/drm/panfrost/panfrost_gpu.h
index 468c51e7e..01a91af09 100644
--- a/drivers/gpu/drm/panfrost/panfrost_gpu.h
+++ b/drivers/gpu/drm/panfrost/panfrost_gpu.h
@@ -18,4 +18,7 @@ void panfrost_gpu_power_off(struct panfrost_device *pfdev);
 
 void panfrost_gpu_amlogic_quirk(struct panfrost_device *pfdev);
 
+void panfrost_acquire_permon(struct panfrost_device *pfdev);
+void panfrost_release_permon(struct panfrost_device *pfdev);
+
 #endif
-- 
2.30.2

From 9ee3e7842e221b87eb761d315b5bcbf517813c55 Mon Sep 17 00:00:00 2001
From: Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>
Date: Thu, 27 May 2021 12:31:05 -0400
Subject: [PATCH 4/6] drm/panfrost: Handle PANFROST_JD_REQ_PERMON

If a job requires cycle counters or timestamps, we must enable cycle
counting just before issuing the job, and disable as soon as the job
completes.

Since this extends the UABI, we bump the driver minor version and date.
That lets userspace detect cycle counter support, and only advertise
features like ARB_shader_clock on kernels with this commit.

Signed-off-by: Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>
---
 drivers/gpu/drm/panfrost/panfrost_drv.c | 10 +++++++---
 drivers/gpu/drm/panfrost/panfrost_job.c |  6 ++++++
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/drivers/gpu/drm/panfrost/panfrost_drv.c b/drivers/gpu/drm/panfrost/panfrost_drv.c
index ca07098a6..0f11d2df4 100644
--- a/drivers/gpu/drm/panfrost/panfrost_drv.c
+++ b/drivers/gpu/drm/panfrost/panfrost_drv.c
@@ -20,6 +20,10 @@
 #include "panfrost_gpu.h"
 #include "panfrost_perfcnt.h"
 
+#define JOB_REQUIREMENTS \
+	(PANFROST_JD_REQ_FS | \
+	 PANFROST_JD_REQ_PERMON)
+
 static bool unstable_ioctls;
 module_param_unsafe(unstable_ioctls, bool, 0600);
 
@@ -247,7 +251,7 @@ static int panfrost_ioctl_submit(struct drm_device *dev, void *data,
 	if (!args->jc)
 		return -EINVAL;
 
-	if (args->requirements && args->requirements != PANFROST_JD_REQ_FS)
+	if (args->requirements & ~JOB_REQUIREMENTS)
 		return -EINVAL;
 
 	if (args->out_sync > 0) {
@@ -557,9 +561,9 @@ static const struct drm_driver panfrost_drm_driver = {
 	.fops			= &panfrost_drm_driver_fops,
 	.name			= "panfrost",
 	.desc			= "panfrost DRM",
-	.date			= "20180908",
+	.date			= "20210527",
 	.major			= 1,
-	.minor			= 1,
+	.minor			= 2,
 
 	.gem_create_object	= panfrost_gem_create_object,
 	.prime_handle_to_fd	= drm_gem_prime_handle_to_fd,
diff --git a/drivers/gpu/drm/panfrost/panfrost_job.c b/drivers/gpu/drm/panfrost/panfrost_job.c
index 6003cfeb1..b78147e3d 100644
--- a/drivers/gpu/drm/panfrost/panfrost_job.c
+++ b/drivers/gpu/drm/panfrost/panfrost_job.c
@@ -165,6 +165,9 @@ static void panfrost_job_hw_submit(struct panfrost_job *job, int js)
 		return;
 	}
 
+	if (job->requirements & PANFROST_JD_REQ_PERMON)
+		panfrost_acquire_permon(job->pfdev);
+
 	cfg = panfrost_mmu_as_get(pfdev, &job->file_priv->mmu);
 
 	job_write(pfdev, JS_HEAD_NEXT_LO(js), jc_head & 0xFFFFFFFF);
@@ -296,6 +299,9 @@ static void panfrost_job_cleanup(struct kref *ref)
 		kvfree(job->bos);
 	}
 
+	if (job->requirements & PANFROST_JD_REQ_PERMON)
+		panfrost_release_permon(job->pfdev);
+
 	kfree(job);
 }
 
-- 
2.30.2

From e29ca72e99d72180241458e3ceb82ccd02bc2986 Mon Sep 17 00:00:00 2001
From: Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>
Date: Fri, 28 May 2021 09:53:27 -0400
Subject: [PATCH 5/6] drm/panfrost: Add GET_PARAM selector for GPU timestamp

Extend the GET_PARAM uABI with a new TIMESTAMP selector, which queries
current GPU time without regard to pending work. Userspace requires this
to implement glGetInteger64v(GL_TIMESTAMP), which has the semantic of
blocking the CPU on the read without submitting anything else, a match
for an ioctl read to the underlying TIMESTAMP register.

At this time we do not add a CYCLE_COUNT selector, as it is not required
by a current API, and its semantics are not well-defined.

Signed-off-by: Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>
---
 include/uapi/drm/panfrost_drm.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/include/uapi/drm/panfrost_drm.h b/include/uapi/drm/panfrost_drm.h
index 27e6cb941..d0e41b5d9 100644
--- a/include/uapi/drm/panfrost_drm.h
+++ b/include/uapi/drm/panfrost_drm.h
@@ -172,6 +172,7 @@ enum drm_panfrost_param {
 	DRM_PANFROST_PARAM_JS_FEATURES15,
 	DRM_PANFROST_PARAM_NR_CORE_GROUPS,
 	DRM_PANFROST_PARAM_THREAD_TLS_ALLOC,
+	DRM_PANFROST_PARAM_TIMESTAMP,
 };
 
 struct drm_panfrost_get_param {
-- 
2.30.2

From 4a0ae1a972c9477cc253a21785e616bd492e07a9 Mon Sep 17 00:00:00 2001
From: Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>
Date: Fri, 28 May 2021 10:26:11 -0400
Subject: [PATCH 6/6] drm/panfrost: Implement DRM_PANFROST_PARAM_TIMESTAMP

Handle DRM_PANFROST_PARAM_TIMESTAMP by reading the 64-bit TIMESTAMP
register after enabling cycle counters. mali_kbase indicates there are
some hardware quirks which make this nontrivial: we must wait for the
cycle counters to actually come online, and we must read repeatedly to
ensure coherent output.

Signed-off-by: Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>
---
 drivers/gpu/drm/panfrost/panfrost_drv.c |  4 +++
 drivers/gpu/drm/panfrost/panfrost_gpu.c | 40 +++++++++++++++++++++++++
 drivers/gpu/drm/panfrost/panfrost_gpu.h |  1 +
 3 files changed, 45 insertions(+)

diff --git a/drivers/gpu/drm/panfrost/panfrost_drv.c b/drivers/gpu/drm/panfrost/panfrost_drv.c
index 0f11d2df4..889f6af49 100644
--- a/drivers/gpu/drm/panfrost/panfrost_drv.c
+++ b/drivers/gpu/drm/panfrost/panfrost_drv.c
@@ -71,6 +71,10 @@ static int panfrost_ioctl_get_param(struct drm_device *ddev, void *data, struct
 		PANFROST_FEATURE_ARRAY(JS_FEATURES, js_features, 15);
 		PANFROST_FEATURE(NR_CORE_GROUPS, nr_core_groups);
 		PANFROST_FEATURE(THREAD_TLS_ALLOC, thread_tls_alloc);
+
+		case DRM_PANFROST_PARAM_TIMESTAMP:
+			return panfrost_query_timestamp(pfdev, &param->value);
+
 	default:
 		return -EINVAL;
 	}
diff --git a/drivers/gpu/drm/panfrost/panfrost_gpu.c b/drivers/gpu/drm/panfrost/panfrost_gpu.c
index acacceb15..e4a894681 100644
--- a/drivers/gpu/drm/panfrost/panfrost_gpu.c
+++ b/drivers/gpu/drm/panfrost/panfrost_gpu.c
@@ -419,3 +419,43 @@ void panfrost_release_permon(struct panfrost_device *pfdev)
 	/* Otherwise, we're the last user */
 	gpu_write(pfdev, GPU_CMD, GPU_CMD_CYCLE_COUNT_STOP);
 }
+
+static int wait_for_cycle_counter(struct panfrost_device *pfdev)
+{
+	int ret;
+	u32 val;
+
+	ret = readl_relaxed_poll_timeout_atomic(pfdev->iomem + GPU_STATUS,
+		val, !(val & GPU_STATUS_CYCLE_COUNT_ACTIVE), 10, 1000);
+
+	if (ret)
+		dev_err(pfdev->dev, "cycle counters stuck inactive");
+
+	return ret;
+}
+
+int panfrost_query_timestamp(struct panfrost_device *pfdev, uint64_t *out)
+{
+	int err;
+	u32 lo = 0, hi = 0;
+
+	panfrost_acquire_permon(pfdev);
+
+	/* Don't read the timestamp until the clocks are ticking */
+	err = wait_for_cycle_counter(pfdev);
+
+	if (err)
+		goto release;
+
+	/* Read hi/lo/hi until we sync. Workaround from mali_kbase */
+	do {
+		hi = gpu_read(pfdev, GPU_TIMESTAMP_HI);
+		lo = gpu_read(pfdev, GPU_TIMESTAMP_LO);
+	} while(hi != gpu_read(pfdev, GPU_TIMESTAMP_HI));
+
+release:
+	*out = (((u64) hi) << 32) | lo;
+
+	panfrost_release_permon(pfdev);
+	return err;
+}
diff --git a/drivers/gpu/drm/panfrost/panfrost_gpu.h b/drivers/gpu/drm/panfrost/panfrost_gpu.h
index 01a91af09..bbf5fec07 100644
--- a/drivers/gpu/drm/panfrost/panfrost_gpu.h
+++ b/drivers/gpu/drm/panfrost/panfrost_gpu.h
@@ -20,5 +20,6 @@ void panfrost_gpu_amlogic_quirk(struct panfrost_device *pfdev);
 
 void panfrost_acquire_permon(struct panfrost_device *pfdev);
 void panfrost_release_permon(struct panfrost_device *pfdev);
+int panfrost_query_timestamp(struct panfrost_device *pfdev, uint64_t *out);
 
 #endif
-- 
2.30.2

