Add Wayland screen capture backend

Author: jonoomph

.gitlab-ci.yml

@@ -41,7 +41,8 @@ linux-builder:
     - export LD_LIBRARY_PATH="$OPENCV_ROOT/lib:$LD_LIBRARY_PATH"
     - export PATH="$OPENCV_ROOT/bin:$PATH"
     - mkdir -p build; cd build;
-    - cmake -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR/build/install-x64" -D"OpenCV_DIR:PATH=$OpenCV_DIR" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -DCMAKE_BUILD_TYPE:STRING=Release -DAPPIMAGE_BUILD=1 -DUSE_SYSTEM_JSONCPP=0 ../
+    - pkg-config --exists libpipewire-0.3 libspa-0.2 gio-2.0 gio-unix-2.0
+    - cmake -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR/build/install-x64" -D"OpenCV_DIR:PATH=$OpenCV_DIR" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -DCMAKE_BUILD_TYPE:STRING=Release -DAPPIMAGE_BUILD=1 -DUSE_SYSTEM_JSONCPP=0 -DENABLE_WAYLAND_CAPTURE=ON ../
     - cmake --build . --parallel $(nproc)
     - make install
     - ctest --output-on-failure -VV

CMakeLists.txt

@@ -72,6 +72,7 @@ option(USE_SYSTEM_JSONCPP "Use system installed JsonCpp, if found" ON)
 option(DISABLE_BUNDLED_JSONCPP "Don't fall back to bundled JsonCpp" OFF)
 option(ENABLE_MAGICK "Use ImageMagick, if available" ON)
 option(ENABLE_OPENCV "Build with OpenCV algorithms (requires Boost, Protobuf 3)" ON)
+option(ENABLE_WAYLAND_CAPTURE "Build Wayland screen capture support with xdg-desktop-portal and PipeWire (Linux only)" ON)
 option(USE_HW_ACCEL "Enable hardware-accelerated encoding-decoding with FFmpeg 3.4+" ON)
 option(ENABLE_VULKAN_BENCHMARK "Build the experimental Vulkan benchmark example" OFF)
 

doc/INSTALL-LINUX.md

@@ -34,6 +34,11 @@ list below to help distinguish between them.
   * http://www.ffmpeg.org/ `(Library)`
   * This library is used to decode and encode video, audio, and image files.  It is also used to obtain information about media files, such as frame rate, sample rate, aspect ratio, and other common attributes.
 
+### PipeWire and xdg-desktop-portal (libpipewire, libspa, GIO)
+  * https://pipewire.org/ `(Library)`
+  * https://flatpak.github.io/xdg-desktop-portal/ `(Runtime Service)`
+  * These libraries and services are used for Wayland screen capture. The development packages are needed at build time, and a desktop portal implementation must be available at runtime.
+
 ### ImageMagick++ (libMagick++, libMagickWand, libMagickCore)
   * http://www.imagemagick.org/script/magick++.php `(Library)`
   * This library is **optional**, and used to decode and encode images.
@@ -154,6 +159,8 @@ software packages available to download and install.
                         libjsoncpp-dev \
                         libmagick++-dev \
                         libopenshot-audio-dev \
+                        libpipewire-0.3-dev \
+                        libspa-0.2-dev \
                         libswscale-dev \
                         libunittest++-dev \
                         libxcursor-dev \
@@ -165,6 +172,8 @@ software packages available to download and install.
                         qtbase5-dev \
                         qtmultimedia5-dev \
                         swig \
+                        xdg-desktop-portal \
+                        xdg-desktop-portal-gtk \
                         python3-zmq \
                         python3-pyqt5.qtwebengine
                         

src/CMakeLists.txt

@@ -95,6 +95,21 @@ set(OPENSHOT_SOURCES
   ZmqLogger.cpp
   )
 
+set(OPENSHOT_WAYLAND_CAPTURE FALSE)
+if(ENABLE_WAYLAND_CAPTURE AND CMAKE_SYSTEM_NAME STREQUAL "Linux")
+  find_package(PkgConfig)
+  if(PkgConfig_FOUND)
+    pkg_check_modules(PC_PIPEWIRE QUIET IMPORTED_TARGET libpipewire-0.3)
+    pkg_check_modules(PC_LIBSPA QUIET IMPORTED_TARGET libspa-0.2)
+    pkg_check_modules(PC_GIO QUIET IMPORTED_TARGET gio-2.0)
+    pkg_check_modules(PC_GIO_UNIX QUIET IMPORTED_TARGET gio-unix-2.0)
+    if(PC_PIPEWIRE_FOUND AND PC_LIBSPA_FOUND AND PC_GIO_FOUND AND PC_GIO_UNIX_FOUND)
+      set(OPENSHOT_WAYLAND_CAPTURE TRUE)
+      list(APPEND OPENSHOT_SOURCES WaylandScreenCaptureReader.cpp)
+    endif()
+  endif()
+endif()
+
 # OpenCV related classes
 set(OPENSHOT_CV_SOURCES
   CVTracker.cpp
@@ -195,6 +210,16 @@ target_include_directories(openshot
     $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
     $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}/libopenshot>)
 
+if(OPENSHOT_WAYLAND_CAPTURE)
+  target_compile_definitions(openshot PRIVATE HAVE_WAYLAND_CAPTURE=1)
+  target_link_libraries(openshot PRIVATE
+    PkgConfig::PC_PIPEWIRE
+    PkgConfig::PC_LIBSPA
+    PkgConfig::PC_GIO
+    PkgConfig::PC_GIO_UNIX)
+endif()
+add_feature_info("Wayland screen capture" OPENSHOT_WAYLAND_CAPTURE "Use xdg-desktop-portal ScreenCast and PipeWire")
+
 ################# LIBOPENSHOT-AUDIO ###################
 # Find JUCE-based openshot Audio libraries
 if(NOT TARGET OpenShot::Audio)

src/ScreenCaptureReader.cpp

@@ -41,8 +41,15 @@ namespace
 	}
 }
 
+#if defined(HAVE_WAYLAND_CAPTURE)
+std::unique_ptr<ScreenCaptureReader::CaptureBackendReader> CreateWaylandScreenCaptureReader(
+	const ScreenCaptureSettings& settings,
+	ReaderInfo& info);
+#endif
+
 ScreenCaptureReader::ScreenCaptureReader(const ScreenCaptureSettings& new_settings)
 	: settings(new_settings)
+	, backend_reader(nullptr)
 	, is_open(false)
 	, video_stream(-1)
 	, frames_read(0)
@@ -59,17 +66,40 @@ ScreenCaptureReader::ScreenCaptureReader(const ScreenCaptureSettings& new_settin
 	}
 	ValidateSettings();
 	PopulateInfo();
+	if (UsesWaylandPortal()) {
+	#if defined(HAVE_WAYLAND_CAPTURE)
+		backend_reader = CreateWaylandScreenCaptureReader(settings, info);
+		if (!backend_reader) {
+			throw InvalidOptions("Wayland screen capture backend is unavailable in this build.");
+		}
+	#else
+		throw InvalidOptions("Wayland screen capture backend is unavailable in this build.");
+	#endif
+	}
 }
 
 ScreenCaptureReader::~ScreenCaptureReader()
 {
 	Close();
 }
 
+bool ScreenCaptureReader::IsOpen()
+{
+	return backend_reader ? backend_reader->IsOpen() : is_open;
+}
+
 bool ScreenCaptureReader::IsBackendSupported(ScreenCaptureBackend backend)
 {
 #if defined(__linux__)
-	return backend == SCREEN_CAPTURE_X11 || backend == SCREEN_CAPTURE_AUTO;
+	if (backend == SCREEN_CAPTURE_X11 || backend == SCREEN_CAPTURE_AUTO) {
+		return true;
+	}
+#if defined(HAVE_WAYLAND_CAPTURE)
+	if (backend == SCREEN_CAPTURE_WAYLAND) {
+		return true;
+	}
+#endif
+	return false;
 #else
 	(void) backend;
 	return false;
@@ -80,6 +110,9 @@ ScreenCaptureBackend ScreenCaptureReader::DefaultBackend()
 {
 #if defined(__linux__)
 	const char* session = std::getenv("XDG_SESSION_TYPE");
+	if (session && std::string(session) == "wayland" && IsBackendSupported(SCREEN_CAPTURE_WAYLAND)) {
+		return SCREEN_CAPTURE_WAYLAND;
+	}
 	if (!session || std::string(session) == "x11") {
 		return SCREEN_CAPTURE_X11;
 	}
@@ -93,7 +126,7 @@ void ScreenCaptureReader::ValidateSettings() const
 	if (!IsBackendSupported(settings.backend)) {
 		throw InvalidOptions("Screen capture backend is not supported on this OS or session.");
 	}
-	if (settings.backend != SCREEN_CAPTURE_X11 && !using_v4l2_device) {
+	if (!UsesFFmpegDevice() && !UsesWaylandPortal() && !using_v4l2_device) {
 		throw InvalidOptions("Only the X11 screen capture backend is implemented in this build.");
 	}
 	if (settings.width <= 0 || settings.height <= 0) {
@@ -104,6 +137,17 @@ void ScreenCaptureReader::ValidateSettings() const
 	}
 }
 
+bool ScreenCaptureReader::UsesFFmpegDevice() const
+{
+	const bool using_v4l2_device = settings.options.count("input_format_name") && settings.options.at("input_format_name") == "v4l2";
+	return settings.backend == SCREEN_CAPTURE_X11 || using_v4l2_device;
+}
+
+bool ScreenCaptureReader::UsesWaylandPortal() const
+{
+	return settings.backend == SCREEN_CAPTURE_WAYLAND;
+}
+
 std::string ScreenCaptureReader::InputFormatName() const
 {
 	const auto override_format = settings.options.find("input_format_name");
@@ -168,6 +212,10 @@ void ScreenCaptureReader::PopulateInfo()
 
 void ScreenCaptureReader::Open()
 {
+	if (backend_reader) {
+		backend_reader->Open();
+		return;
+	}
 	if (is_open) {
 		return;
 	}
@@ -260,6 +308,13 @@ void ScreenCaptureReader::OpenDecoder()
 
 std::shared_ptr<Frame> ScreenCaptureReader::GetFrame(int64_t number)
 {
+	if (backend_reader) {
+		if (!backend_reader->IsOpen()) {
+			throw ReaderClosed("The ScreenCaptureReader is closed. Call Open() before GetFrame().");
+		}
+		const std::lock_guard<std::recursive_mutex> lock(getFrameMutex);
+		return backend_reader->GetFrame(number);
+	}
 	if (!is_open) {
 		throw ReaderClosed("The ScreenCaptureReader is closed. Call Open() before GetFrame().");
 	}
@@ -337,6 +392,9 @@ std::shared_ptr<Frame> ScreenCaptureReader::DecodeNextFrame(int64_t number)
 
 void ScreenCaptureReader::Close()
 {
+	if (backend_reader) {
+		backend_reader->Close();
+	}
 	if (packet) {
 		av_packet_free(&packet);
 	}
@@ -362,6 +420,9 @@ void ScreenCaptureReader::Close()
 
 openshot::CaptureReaderStats ScreenCaptureReader::GetStats() const
 {
+	if (backend_reader) {
+		return backend_reader->GetStats();
+	}
 	CaptureReaderStats stats;
 	stats.is_open = is_open;
 	stats.frames_read = frames_read;

src/ScreenCaptureReader.h

@@ -57,13 +57,26 @@ namespace openshot
 	class ScreenCaptureReader : public ReaderBase
 	{
 	public:
+#ifndef SWIG
+		class CaptureBackendReader
+		{
+		public:
+			virtual ~CaptureBackendReader() = default;
+			virtual void Open() = 0;
+			virtual void Close() = 0;
+			virtual bool IsOpen() const = 0;
+			virtual std::shared_ptr<openshot::Frame> GetFrame(int64_t number) = 0;
+			virtual openshot::CaptureReaderStats GetStats() const = 0;
+		};
+#endif
+
 		explicit ScreenCaptureReader(const ScreenCaptureSettings& settings);
 		~ScreenCaptureReader() override;
 
 		void Close() override;
 		CacheBase* GetCache() override { return NULL; };
 		std::shared_ptr<openshot::Frame> GetFrame(int64_t number) override;
-		bool IsOpen() override { return is_open; };
+		bool IsOpen() override;
 		std::string Name() override { return "ScreenCaptureReader"; };
 		std::string Json() const override;
 		void SetJson(const std::string value) override;
@@ -83,9 +96,14 @@ namespace openshot
 		std::string InputFormatName() const;
 		std::string InputName() const;
 		std::shared_ptr<openshot::Frame> DecodeNextFrame(int64_t number);
+		bool UsesFFmpegDevice() const;
+		bool UsesWaylandPortal() const;
 		void PopulateInfo();
 
 		ScreenCaptureSettings settings;
+#ifndef SWIG
+		std::unique_ptr<CaptureBackendReader> backend_reader;
+#endif
 		bool is_open;
 		int video_stream;
 		int64_t frames_read;