Initial commit: RustDesk 1.4.7 macOS desktop port

- Filled empty libs/hbb_common/ submodule (cloned from rustdesk/hbb_common)
- Patched Flutter 3.44 / Dart 3.12 compatibility:
  * flutter/lib/generated_bridge.dart: asTypedList with cast<>, DartPort=Int64
  * flutter/lib/common.dart: DialogTheme->DialogThemeData, TabBarTheme->TabBarThemeData
  * flutter/pubspec.yaml: extended_text 14.0.0->15.0.2, google_fonts override 5.0.0
  * flutter/macos/Runner/Configs/Release.xcconfig: EXCLUDED_ARCHS=x86_64
- Build verified: cargo check + cargo build --features flutter + cargo build --release --features flutter
- Verified flutter build macos --debug and --release both produce working .app
- Verified .dmg installer (27MB arm64) created via hdiutil
- Build deps: Xcode 26.5, CocoaPods 1.16.2, Flutter 3.44, VCPKG arm64-osx

libs/scrap/src/common/android.rs

@@ -0,0 +1,189 @@
+use crate::android::ffi::*;
+use crate::{Frame, Pixfmt};
+use lazy_static::lazy_static;
+use serde_json::Value;
+use std::collections::HashMap;
+use std::sync::Mutex;
+use std::{io, time::Duration};
+
+lazy_static! {
+   pub(crate)  static ref SCREEN_SIZE: Mutex<(u16, u16, u16)> = Mutex::new((0, 0, 0)); // (width, height, scale)
+}
+
+pub struct Capturer {
+    display: Display,
+    rgba: Vec<u8>,
+    saved_raw_data: Vec<u8>, // for faster compare and copy
+}
+
+impl Capturer {
+    pub fn new(display: Display) -> io::Result<Capturer> {
+        Ok(Capturer {
+            display,
+            rgba: Vec::new(),
+            saved_raw_data: Vec::new(),
+        })
+    }
+
+    pub fn width(&self) -> usize {
+        self.display.width() as usize
+    }
+
+    pub fn height(&self) -> usize {
+        self.display.height() as usize
+    }
+}
+
+impl crate::TraitCapturer for Capturer {
+    fn frame<'a>(&'a mut self, _timeout: Duration) -> io::Result<Frame<'a>> {
+        if get_video_raw(&mut self.rgba, &mut self.saved_raw_data).is_some() {
+            Ok(Frame::PixelBuffer(PixelBuffer::new(
+                &self.rgba,
+                self.width(),
+                self.height(),
+            )))
+        } else {
+            return Err(io::ErrorKind::WouldBlock.into());
+        }
+    }
+}
+
+pub struct PixelBuffer<'a> {
+    data: &'a [u8],
+    width: usize,
+    height: usize,
+    stride: Vec<usize>,
+}
+
+impl<'a> PixelBuffer<'a> {
+    pub fn new(data: &'a [u8], width: usize, height: usize) -> Self {
+        let stride0 = data.len() / height;
+        let mut stride = Vec::new();
+        stride.push(stride0);
+        PixelBuffer {
+            data,
+            width,
+            height,
+            stride,
+        }
+    }
+}
+
+impl<'a> crate::TraitPixelBuffer for PixelBuffer<'a> {
+    fn data(&self) -> &[u8] {
+        self.data
+    }
+
+    fn width(&self) -> usize {
+        self.width
+    }
+
+    fn height(&self) -> usize {
+        self.height
+    }
+
+    fn stride(&self) -> Vec<usize> {
+        self.stride.clone()
+    }
+
+    fn pixfmt(&self) -> Pixfmt {
+        Pixfmt::RGBA
+    }
+}
+
+pub struct Display {
+    default: bool,
+    rect: Rect,
+}
+
+#[derive(Copy, Clone, Debug, Hash, Eq, PartialEq)]
+struct Rect {
+    pub x: i16,
+    pub y: i16,
+    pub w: u16,
+    pub h: u16,
+}
+
+impl Display {
+    pub fn primary() -> io::Result<Display> {
+        let mut size = SCREEN_SIZE.lock().unwrap();
+        if size.0 == 0 || size.1 == 0 {
+            *size = get_size().unwrap_or_default();
+        }
+        Ok(Display {
+            default: true,
+            rect: Rect {
+                x: 0,
+                y: 0,
+                w: size.0,
+                h: size.1,
+            },
+        })
+    }
+
+    pub fn all() -> io::Result<Vec<Display>> {
+        Ok(vec![Display::primary()?])
+    }
+
+    pub fn width(&self) -> usize {
+        self.rect.w as usize
+    }
+
+    pub fn height(&self) -> usize {
+        self.rect.h as usize
+    }
+
+    pub fn origin(&self) -> (i32, i32) {
+        let r = self.rect;
+        (r.x as _, r.y as _)
+    }
+
+    pub fn is_online(&self) -> bool {
+        true
+    }
+
+    pub fn is_primary(&self) -> bool {
+        self.default
+    }
+
+    pub fn name(&self) -> String {
+        "Android".into()
+    }
+
+    pub fn refresh_size() {
+        let mut size = SCREEN_SIZE.lock().unwrap();
+        *size = get_size().unwrap_or_default();
+    }
+
+    // Big android screen size will be shrinked, to improve performance when screen-capturing and encoding
+    // e.g 2280x1080 size will be set to 1140x540, and `scale` is 2
+    // need to multiply by `4` (2*2) when compute the bitrate
+    pub fn fix_quality() -> u16 {
+        let scale = SCREEN_SIZE.lock().unwrap().2;
+        if scale <= 0 {
+            1
+        } else {
+            scale * scale
+        }
+    }
+}
+
+fn get_size() -> Option<(u16, u16, u16)> {
+    let res = call_main_service_get_by_name("screen_size").ok()?;
+    if let Ok(json) = serde_json::from_str::<HashMap<String, Value>>(&res) {
+        if let (Some(Value::Number(w)), Some(Value::Number(h)), Some(Value::Number(scale))) =
+            (json.get("width"), json.get("height"), json.get("scale"))
+        {
+            let w = w.as_i64()? as _;
+            let h = h.as_i64()? as _;
+            let scale = scale.as_i64()? as _;
+            return Some((w, h, scale));
+        }
+    }
+    None
+}
+
+pub fn is_start() -> Option<bool> {
+    let res = call_main_service_get_by_name("is_start").ok()?;
+    Some(res == "true")
+}

libs/scrap/src/common/aom.rs

@@ -0,0 +1,581 @@
+#![allow(non_camel_case_types)]
+#![allow(non_snake_case)]
+#![allow(non_upper_case_globals)]
+#![allow(improper_ctypes)]
+#![allow(dead_code)]
+
+include!(concat!(env!("OUT_DIR"), "/aom_ffi.rs"));
+
+use crate::codec::{base_bitrate, codec_thread_num};
+use crate::{codec::EncoderApi, EncodeFrame, STRIDE_ALIGN};
+use crate::{common::GoogleImage, generate_call_macro, generate_call_ptr_macro, Error, Result};
+use crate::{EncodeInput, EncodeYuvFormat, Pixfmt};
+use hbb_common::{
+    anyhow::{anyhow, Context},
+    bytes::Bytes,
+    log,
+    message_proto::{Chroma, EncodedVideoFrame, EncodedVideoFrames, VideoFrame},
+    ResultType,
+};
+use std::{ptr, slice};
+
+generate_call_macro!(call_aom, false);
+generate_call_macro!(call_aom_allow_err, true);
+generate_call_ptr_macro!(call_aom_ptr);
+
+impl Default for aom_codec_enc_cfg_t {
+    fn default() -> Self {
+        unsafe { std::mem::zeroed() }
+    }
+}
+
+impl Default for aom_codec_ctx_t {
+    fn default() -> Self {
+        unsafe { std::mem::zeroed() }
+    }
+}
+
+impl Default for aom_image_t {
+    fn default() -> Self {
+        unsafe { std::mem::zeroed() }
+    }
+}
+
+#[derive(Clone, Copy, Debug)]
+pub struct AomEncoderConfig {
+    pub width: u32,
+    pub height: u32,
+    pub quality: f32,
+    pub keyframe_interval: Option<usize>,
+}
+
+pub struct AomEncoder {
+    ctx: aom_codec_ctx_t,
+    width: usize,
+    height: usize,
+    i444: bool,
+    yuvfmt: EncodeYuvFormat,
+}
+
+// https://webrtc.googlesource.com/src/+/refs/heads/main/modules/video_coding/codecs/av1/libaom_av1_encoder.cc
+mod webrtc {
+    use super::*;
+
+    const kUsageProfile: u32 = AOM_USAGE_REALTIME;
+    const kBitDepth: u32 = 8;
+    const kLagInFrames: u32 = 0; // No look ahead.
+    pub(super) const kTimeBaseDen: i64 = 1000;
+
+    // Only positive speeds, range for real-time coding currently is: 6 - 8.
+    // Lower means slower/better quality, higher means fastest/lower quality.
+    fn get_cpu_speed(width: u32, height: u32) -> u32 {
+        // aux_config_ = nullptr, kComplexityHigh
+        if width * height <= 320 * 180 {
+            8
+        } else if width * height <= 640 * 360 {
+            9
+        } else {
+            10
+        }
+    }
+
+    fn get_super_block_size(width: u32, height: u32, threads: u32) -> aom_superblock_size_t {
+        use aom_superblock_size::*;
+        let resolution = width * height;
+        if threads >= 4 && resolution >= 960 * 540 && resolution < 1920 * 1080 {
+            AOM_SUPERBLOCK_SIZE_64X64
+        } else {
+            AOM_SUPERBLOCK_SIZE_DYNAMIC
+        }
+    }
+
+    pub fn enc_cfg(
+        i: *const aom_codec_iface,
+        cfg: AomEncoderConfig,
+        i444: bool,
+    ) -> ResultType<aom_codec_enc_cfg> {
+        let mut c = unsafe { std::mem::MaybeUninit::zeroed().assume_init() };
+        call_aom!(aom_codec_enc_config_default(i, &mut c, kUsageProfile));
+
+        // Overwrite default config with input encoder settings & RTC-relevant values.
+        c.g_w = cfg.width;
+        c.g_h = cfg.height;
+        c.g_threads = codec_thread_num(64) as _;
+        c.g_timebase.num = 1;
+        c.g_timebase.den = kTimeBaseDen as _;
+        c.g_input_bit_depth = kBitDepth;
+        if let Some(keyframe_interval) = cfg.keyframe_interval {
+            c.kf_min_dist = 0;
+            c.kf_max_dist = keyframe_interval as _;
+        } else {
+            c.kf_mode = aom_kf_mode::AOM_KF_DISABLED;
+        }
+        let (q_min, q_max) = AomEncoder::calc_q_values(cfg.quality);
+        c.rc_min_quantizer = q_min;
+        c.rc_max_quantizer = q_max;
+        c.rc_target_bitrate = AomEncoder::bitrate(cfg.width as _, cfg.height as _, cfg.quality);
+        c.rc_undershoot_pct = 50;
+        c.rc_overshoot_pct = 50;
+        c.rc_buf_initial_sz = 600;
+        c.rc_buf_optimal_sz = 600;
+        c.rc_buf_sz = 1000;
+        c.g_usage = kUsageProfile;
+        c.g_error_resilient = 0;
+        // Low-latency settings.
+        c.rc_end_usage = aom_rc_mode::AOM_CBR; // Constant Bit Rate (CBR) mode
+        c.g_pass = aom_enc_pass::AOM_RC_ONE_PASS; // One-pass rate control
+        c.g_lag_in_frames = kLagInFrames; // No look ahead when lag equals 0.
+
+        // https://aomedia.googlesource.com/aom/+/refs/tags/v3.6.0/av1/common/enums.h#82
+        c.g_profile = if i444 { 1 } else { 0 };
+
+        Ok(c)
+    }
+
+    pub fn set_controls(ctx: *mut aom_codec_ctx_t, cfg: &aom_codec_enc_cfg) -> ResultType<()> {
+        use aom_tune_content::*;
+        use aome_enc_control_id::*;
+        macro_rules! call_ctl {
+            ($ctx:expr, $av1e:expr, $arg:expr) => {{
+                call_aom_allow_err!(aom_codec_control($ctx, $av1e as i32, $arg));
+            }};
+        }
+
+        call_ctl!(ctx, AOME_SET_CPUUSED, get_cpu_speed(cfg.g_w, cfg.g_h));
+        call_ctl!(ctx, AV1E_SET_ENABLE_CDEF, 1);
+        call_ctl!(ctx, AV1E_SET_ENABLE_TPL_MODEL, 0);
+        call_ctl!(ctx, AV1E_SET_DELTAQ_MODE, 0);
+        call_ctl!(ctx, AV1E_SET_ENABLE_ORDER_HINT, 0);
+        call_ctl!(ctx, AV1E_SET_AQ_MODE, 3);
+        call_ctl!(ctx, AOME_SET_MAX_INTRA_BITRATE_PCT, 300);
+        call_ctl!(ctx, AV1E_SET_COEFF_COST_UPD_FREQ, 3);
+        call_ctl!(ctx, AV1E_SET_MODE_COST_UPD_FREQ, 3);
+        call_ctl!(ctx, AV1E_SET_MV_COST_UPD_FREQ, 3);
+        // kScreensharing
+        call_ctl!(ctx, AV1E_SET_TUNE_CONTENT, AOM_CONTENT_SCREEN);
+        call_ctl!(ctx, AV1E_SET_ENABLE_PALETTE, 1);
+        let tile_set = if cfg.g_threads == 4 && cfg.g_w == 640 && (cfg.g_h == 360 || cfg.g_h == 480)
+        {
+            AV1E_SET_TILE_ROWS
+        } else {
+            AV1E_SET_TILE_COLUMNS
+        };
+        // Failed on android
+        call_ctl!(ctx, tile_set, (cfg.g_threads as f64 * 1.0f64).log2().ceil());
+        call_ctl!(ctx, AV1E_SET_ROW_MT, 1);
+        call_ctl!(ctx, AV1E_SET_ENABLE_OBMC, 0);
+        call_ctl!(ctx, AV1E_SET_NOISE_SENSITIVITY, 0);
+        call_ctl!(ctx, AV1E_SET_ENABLE_WARPED_MOTION, 0);
+        call_ctl!(ctx, AV1E_SET_ENABLE_GLOBAL_MOTION, 0);
+        call_ctl!(ctx, AV1E_SET_ENABLE_REF_FRAME_MVS, 0);
+        call_ctl!(
+            ctx,
+            AV1E_SET_SUPERBLOCK_SIZE,
+            get_super_block_size(cfg.g_w, cfg.g_h, cfg.g_threads)
+        );
+        call_ctl!(ctx, AV1E_SET_ENABLE_CFL_INTRA, 0);
+        call_ctl!(ctx, AV1E_SET_ENABLE_SMOOTH_INTRA, 0);
+        call_ctl!(ctx, AV1E_SET_ENABLE_ANGLE_DELTA, 0);
+        call_ctl!(ctx, AV1E_SET_ENABLE_FILTER_INTRA, 0);
+        call_ctl!(ctx, AV1E_SET_INTRA_DEFAULT_TX_ONLY, 1);
+        call_ctl!(ctx, AV1E_SET_DISABLE_TRELLIS_QUANT, 1);
+        call_ctl!(ctx, AV1E_SET_ENABLE_DIST_WTD_COMP, 0);
+        call_ctl!(ctx, AV1E_SET_ENABLE_DIFF_WTD_COMP, 0);
+        call_ctl!(ctx, AV1E_SET_ENABLE_DUAL_FILTER, 0);
+        call_ctl!(ctx, AV1E_SET_ENABLE_INTERINTRA_COMP, 0);
+        call_ctl!(ctx, AV1E_SET_ENABLE_INTERINTRA_WEDGE, 0);
+        call_ctl!(ctx, AV1E_SET_ENABLE_INTRA_EDGE_FILTER, 0);
+        call_ctl!(ctx, AV1E_SET_ENABLE_INTRABC, 0);
+        call_ctl!(ctx, AV1E_SET_ENABLE_MASKED_COMP, 0);
+        call_ctl!(ctx, AV1E_SET_ENABLE_PAETH_INTRA, 0);
+        call_ctl!(ctx, AV1E_SET_ENABLE_QM, 0);
+        call_ctl!(ctx, AV1E_SET_ENABLE_RECT_PARTITIONS, 0);
+        call_ctl!(ctx, AV1E_SET_ENABLE_RESTORATION, 0);
+        call_ctl!(ctx, AV1E_SET_ENABLE_SMOOTH_INTERINTRA, 0);
+        call_ctl!(ctx, AV1E_SET_ENABLE_TX64, 0);
+        call_ctl!(ctx, AV1E_SET_MAX_REFERENCE_FRAMES, 3);
+
+        Ok(())
+    }
+}
+
+impl EncoderApi for AomEncoder {
+    fn new(cfg: crate::codec::EncoderCfg, i444: bool) -> ResultType<Self>
+    where
+        Self: Sized,
+    {
+        match cfg {
+            crate::codec::EncoderCfg::AOM(config) => {
+                let i = call_aom_ptr!(aom_codec_av1_cx());
+                let c = webrtc::enc_cfg(i, config, i444)?;
+
+                let mut ctx = Default::default();
+                // Flag options: AOM_CODEC_USE_PSNR and AOM_CODEC_USE_HIGHBITDEPTH
+                let flags: aom_codec_flags_t = 0;
+                call_aom!(aom_codec_enc_init_ver(
+                    &mut ctx,
+                    i,
+                    &c,
+                    flags,
+                    AOM_ENCODER_ABI_VERSION as _
+                ));
+                webrtc::set_controls(&mut ctx, &c)?;
+                Ok(Self {
+                    ctx,
+                    width: config.width as _,
+                    height: config.height as _,
+                    i444,
+                    yuvfmt: Self::get_yuvfmt(config.width, config.height, i444),
+                })
+            }
+            _ => Err(anyhow!("encoder type mismatch")),
+        }
+    }
+
+    fn encode_to_message(&mut self, input: EncodeInput, ms: i64) -> ResultType<VideoFrame> {
+        let mut frames = Vec::new();
+        for ref frame in self
+            .encode(ms, input.yuv()?, STRIDE_ALIGN)
+            .with_context(|| "Failed to encode")?
+        {
+            frames.push(Self::create_frame(frame));
+        }
+        if frames.len() > 0 {
+            Ok(Self::create_video_frame(frames))
+        } else {
+            Err(anyhow!("no valid frame"))
+        }
+    }
+
+    fn yuvfmt(&self) -> crate::EncodeYuvFormat {
+        self.yuvfmt.clone()
+    }
+
+    #[cfg(feature = "vram")]
+    fn input_texture(&self) -> bool {
+        false
+    }
+
+    fn set_quality(&mut self, ratio: f32) -> ResultType<()> {
+        let mut c = unsafe { *self.ctx.config.enc.to_owned() };
+        let (q_min, q_max) = Self::calc_q_values(ratio);
+        c.rc_min_quantizer = q_min;
+        c.rc_max_quantizer = q_max;
+        c.rc_target_bitrate = Self::bitrate(self.width as _, self.height as _, ratio);
+        call_aom!(aom_codec_enc_config_set(&mut self.ctx, &c));
+        Ok(())
+    }
+
+    fn bitrate(&self) -> u32 {
+        let c = unsafe { *self.ctx.config.enc.to_owned() };
+        c.rc_target_bitrate
+    }
+
+    fn support_changing_quality(&self) -> bool {
+        true
+    }
+
+    fn latency_free(&self) -> bool {
+        true
+    }
+
+    fn is_hardware(&self) -> bool {
+        false
+    }
+
+    fn disable(&self) {}
+}
+
+impl AomEncoder {
+    pub fn encode<'a>(&'a mut self, ms: i64, data: &[u8], stride_align: usize) -> Result<EncodeFrames<'a>> {
+        let bpp = if self.i444 { 24 } else { 12 };
+        if data.len() < self.width * self.height * bpp / 8 {
+            return Err(Error::FailedCall("len not enough".to_string()));
+        }
+        let fmt = if self.i444 {
+            aom_img_fmt::AOM_IMG_FMT_I444
+        } else {
+            aom_img_fmt::AOM_IMG_FMT_I420
+        };
+
+        let mut image = Default::default();
+        call_aom_ptr!(aom_img_wrap(
+            &mut image,
+            fmt,
+            self.width as _,
+            self.height as _,
+            stride_align as _,
+            data.as_ptr() as _,
+        ));
+        let pts = webrtc::kTimeBaseDen / 1000 * ms;
+        let duration = webrtc::kTimeBaseDen / 1000;
+        call_aom!(aom_codec_encode(
+            &mut self.ctx,
+            &image,
+            pts as _,
+            duration as _, // Duration
+            0,             // Flags
+        ));
+
+        Ok(EncodeFrames {
+            ctx: &mut self.ctx,
+            iter: ptr::null(),
+        })
+    }
+
+    #[inline]
+    pub fn create_video_frame(frames: Vec<EncodedVideoFrame>) -> VideoFrame {
+        let mut vf = VideoFrame::new();
+        let av1s = EncodedVideoFrames {
+            frames: frames.into(),
+            ..Default::default()
+        };
+        vf.set_av1s(av1s);
+        vf
+    }
+
+    #[inline]
+    fn create_frame(frame: &EncodeFrame) -> EncodedVideoFrame {
+        EncodedVideoFrame {
+            data: Bytes::from(frame.data.to_vec()),
+            key: frame.key,
+            pts: frame.pts,
+            ..Default::default()
+        }
+    }
+
+    fn bitrate(width: u32, height: u32, ratio: f32) -> u32 {
+        let bitrate = base_bitrate(width, height) as f32;
+        (bitrate * ratio) as u32
+    }
+
+    #[inline]
+    fn calc_q_values(ratio: f32) -> (u32, u32) {
+        let b = (ratio * 100.0) as u32;
+        let b = std::cmp::min(b, 200);
+        let q_min1 = 24;
+        let q_min2 = 5;
+        let q_max1 = 45;
+        let q_max2 = 25;
+
+        let t = b as f32 / 200.0;
+
+        let mut q_min: u32 = ((1.0 - t) * q_min1 as f32 + t * q_min2 as f32).round() as u32;
+        let mut q_max = ((1.0 - t) * q_max1 as f32 + t * q_max2 as f32).round() as u32;
+
+        q_min = q_min.clamp(q_min2, q_min1);
+        q_max = q_max.clamp(q_max2, q_max1);
+
+        (q_min, q_max)
+    }
+
+    fn get_yuvfmt(width: u32, height: u32, i444: bool) -> EncodeYuvFormat {
+        let mut img = Default::default();
+        let fmt = if i444 {
+            aom_img_fmt::AOM_IMG_FMT_I444
+        } else {
+            aom_img_fmt::AOM_IMG_FMT_I420
+        };
+        unsafe {
+            aom_img_wrap(
+                &mut img,
+                fmt,
+                width as _,
+                height as _,
+                crate::STRIDE_ALIGN as _,
+                0x1 as _,
+            );
+        }
+        let pixfmt = if i444 { Pixfmt::I444 } else { Pixfmt::I420 };
+        EncodeYuvFormat {
+            pixfmt,
+            w: img.w as _,
+            h: img.h as _,
+            stride: img.stride.map(|s| s as usize).to_vec(),
+            u: img.planes[1] as usize - img.planes[0] as usize,
+            v: img.planes[2] as usize - img.planes[0] as usize,
+        }
+    }
+}
+
+impl Drop for AomEncoder {
+    fn drop(&mut self) {
+        unsafe {
+            let result = aom_codec_destroy(&mut self.ctx);
+            if result != aom_codec_err_t::AOM_CODEC_OK {
+                panic!("failed to destroy aom codec");
+            }
+        }
+    }
+}
+
+pub struct EncodeFrames<'a> {
+    ctx: &'a mut aom_codec_ctx_t,
+    iter: aom_codec_iter_t,
+}
+
+impl<'a> Iterator for EncodeFrames<'a> {
+    type Item = EncodeFrame<'a>;
+    fn next(&mut self) -> Option<Self::Item> {
+        loop {
+            unsafe {
+                let pkt = aom_codec_get_cx_data(self.ctx, &mut self.iter);
+                if pkt.is_null() {
+                    return None;
+                } else if (*pkt).kind == aom_codec_cx_pkt_kind::AOM_CODEC_CX_FRAME_PKT {
+                    let f = &(*pkt).data.frame;
+                    return Some(Self::Item {
+                        data: slice::from_raw_parts(f.buf as _, f.sz as _),
+                        key: (f.flags & AOM_FRAME_IS_KEY) != 0,
+                        pts: f.pts,
+                    });
+                } else {
+                    // Ignore the packet.
+                }
+            }
+        }
+    }
+}
+
+pub struct AomDecoder {
+    ctx: aom_codec_ctx_t,
+}
+
+impl AomDecoder {
+    pub fn new() -> Result<Self> {
+        let i = call_aom_ptr!(aom_codec_av1_dx());
+        let mut ctx = Default::default();
+        let cfg = aom_codec_dec_cfg_t {
+            threads: codec_thread_num(64) as _,
+            w: 0,
+            h: 0,
+            allow_lowbitdepth: 1,
+        };
+        call_aom!(aom_codec_dec_init_ver(
+            &mut ctx,
+            i,
+            &cfg,
+            0,
+            AOM_DECODER_ABI_VERSION as _,
+        ));
+        Ok(Self { ctx })
+    }
+
+    pub fn decode<'a>(&'a mut self, data: &[u8]) -> Result<DecodeFrames<'a>> {
+        call_aom!(aom_codec_decode(
+            &mut self.ctx,
+            data.as_ptr(),
+            data.len() as _,
+            ptr::null_mut(),
+        ));
+
+        Ok(DecodeFrames {
+            ctx: &mut self.ctx,
+            iter: ptr::null(),
+        })
+    }
+
+    /// Notify the decoder to return any pending frame
+    pub fn flush<'a>(&'a mut self) -> Result<DecodeFrames<'a>> {
+        call_aom!(aom_codec_decode(
+            &mut self.ctx,
+            ptr::null(),
+            0,
+            ptr::null_mut(),
+        ));
+        Ok(DecodeFrames {
+            ctx: &mut self.ctx,
+            iter: ptr::null(),
+        })
+    }
+}
+
+impl Drop for AomDecoder {
+    fn drop(&mut self) {
+        unsafe {
+            let result = aom_codec_destroy(&mut self.ctx);
+            if result != aom_codec_err_t::AOM_CODEC_OK {
+                panic!("failed to destroy aom codec");
+            }
+        }
+    }
+}
+
+pub struct DecodeFrames<'a> {
+    ctx: &'a mut aom_codec_ctx_t,
+    iter: aom_codec_iter_t,
+}
+
+impl<'a> Iterator for DecodeFrames<'a> {
+    type Item = Image;
+    fn next(&mut self) -> Option<Self::Item> {
+        let img = unsafe { aom_codec_get_frame(self.ctx, &mut self.iter) };
+        if img.is_null() {
+            return None;
+        } else {
+            return Some(Image(img));
+        }
+    }
+}
+
+pub struct Image(*mut aom_image_t);
+impl Image {
+    #[inline]
+    pub fn new() -> Self {
+        Self(std::ptr::null_mut())
+    }
+
+    #[inline]
+    pub fn is_null(&self) -> bool {
+        self.0.is_null()
+    }
+
+    #[inline]
+    pub fn format(&self) -> aom_img_fmt_t {
+        self.inner().fmt
+    }
+
+    #[inline]
+    pub fn inner(&self) -> &aom_image_t {
+        unsafe { &*self.0 }
+    }
+}
+
+impl GoogleImage for Image {
+    #[inline]
+    fn width(&self) -> usize {
+        self.inner().d_w as _
+    }
+
+    #[inline]
+    fn height(&self) -> usize {
+        self.inner().d_h as _
+    }
+
+    #[inline]
+    fn stride(&self) -> Vec<i32> {
+        self.inner().stride.iter().map(|x| *x as i32).collect()
+    }
+
+    #[inline]
+    fn planes(&self) -> Vec<*mut u8> {
+        self.inner().planes.iter().map(|p| *p as *mut u8).collect()
+    }
+
+    fn chroma(&self) -> Chroma {
+        match self.inner().fmt {
+            aom_img_fmt::AOM_IMG_FMT_I444 => Chroma::I444,
+            _ => Chroma::I420,
+        }
+    }
+}
+
+impl Drop for Image {
+    fn drop(&mut self) {
+        if !self.0.is_null() {
+            unsafe { aom_img_free(self.0) };
+        }
+    }
+}
+
+unsafe impl Send for aom_codec_ctx_t {}

libs/scrap/src/common/camera.rs

@@ -0,0 +1,286 @@
+use std::{
+    io,
+    sync::{Arc, Mutex},
+};
+
+#[cfg(any(target_os = "windows", target_os = "linux"))]
+use nokhwa::{
+    pixel_format::RgbAFormat,
+    query,
+    utils::{ApiBackend, CameraIndex, RequestedFormat, RequestedFormatType},
+    Camera,
+};
+
+use hbb_common::message_proto::{DisplayInfo, Resolution};
+
+#[cfg(feature = "vram")]
+use crate::AdapterDevice;
+
+use crate::common::{bail, ResultType};
+use crate::{Frame, TraitCapturer};
+#[cfg(any(target_os = "windows", target_os = "linux"))]
+use crate::{PixelBuffer, Pixfmt};
+
+pub const PRIMARY_CAMERA_IDX: usize = 0;
+lazy_static::lazy_static! {
+    static ref SYNC_CAMERA_DISPLAYS: Arc<Mutex<Vec<DisplayInfo>>> = Arc::new(Mutex::new(Vec::new()));
+}
+
+#[cfg(not(any(target_os = "windows", target_os = "linux")))]
+const CAMERA_NOT_SUPPORTED: &str = "This platform doesn't support camera yet";
+
+pub struct Cameras;
+
+// pre-condition
+pub fn primary_camera_exists() -> bool {
+    Cameras::exists(PRIMARY_CAMERA_IDX)
+}
+
+#[cfg(any(target_os = "windows", target_os = "linux"))]
+impl Cameras {
+    pub fn all_info() -> ResultType<Vec<DisplayInfo>> {
+        match query(ApiBackend::Auto) {
+            Ok(cameras) => {
+                let mut camera_displays = SYNC_CAMERA_DISPLAYS.lock().unwrap();
+                camera_displays.clear();
+                // FIXME: nokhwa returns duplicate info for one physical camera on linux for now.
+                // issue: https://github.com/l1npengtul/nokhwa/issues/171
+                // Use only one camera as a temporary hack.
+                cfg_if::cfg_if! {
+                    if #[cfg(target_os = "linux")] {
+                        let Some(info) = cameras.first() else {
+                            bail!("No camera found")
+                        };
+                        // Use index (0) camera as main camera, fallback to the first camera if index (0) is not available.
+                        // But maybe we also need to check index (1) or the lowest index camera.
+                        //
+                        // https://askubuntu.com/questions/234362/how-to-fix-this-problem-where-sometimes-dev-video0-becomes-automatically-dev
+                        // https://github.com/rustdesk/rustdesk/pull/12010#issue-3125329069
+                        let mut camera_index = info.index().clone();
+                        if !matches!(camera_index, CameraIndex::Index(0)) {
+                            if cameras.iter().any(|cam| matches!(cam.index(), CameraIndex::Index(0))) {
+                                camera_index = CameraIndex::Index(0);
+                            }
+                        }
+                        let camera = Self::create_camera(&camera_index)?;
+                        let resolution = camera.resolution();
+                        let (width, height) = (resolution.width() as i32, resolution.height() as i32);
+                        camera_displays.push(DisplayInfo {
+                            x: 0,
+                            y: 0,
+                            name: info.human_name().clone(),
+                            width,
+                            height,
+                            online: true,
+                            cursor_embedded: false,
+                            scale:1.0,
+                            original_resolution: Some(Resolution {
+                                width,
+                                height,
+                                ..Default::default()
+                            }).into(),
+                            ..Default::default()
+                        });
+                    } else {
+                        let mut x = 0;
+                        for info in &cameras {
+                            let camera = Self::create_camera(info.index())?;
+                            let resolution = camera.resolution();
+                            let (width, height) = (resolution.width() as i32, resolution.height() as i32);
+                            camera_displays.push(DisplayInfo {
+                                x,
+                                y: 0,
+                                name: info.human_name().clone(),
+                                width,
+                                height,
+                                online: true,
+                                cursor_embedded: false,
+                                scale:1.0,
+                                original_resolution: Some(Resolution {
+                                    width,
+                                    height,
+                                    ..Default::default()
+                                }).into(),
+                                ..Default::default()
+                            });
+                            x += width;
+                        }
+                    }
+                }
+                Ok(camera_displays.clone())
+            }
+            Err(e) => {
+                bail!("Query cameras error: {}", e)
+            }
+        }
+    }
+
+    pub fn exists(index: usize) -> bool {
+        match query(ApiBackend::Auto) {
+            Ok(cameras) => index < cameras.len(),
+            _ => return false,
+        }
+    }
+
+    fn create_camera(index: &CameraIndex) -> ResultType<Camera> {
+        let format_type = if cfg!(target_os = "linux") {
+            RequestedFormatType::None
+        } else {
+            RequestedFormatType::AbsoluteHighestResolution
+        };
+        let result = Camera::new(
+            index.clone(),
+            RequestedFormat::new::<RgbAFormat>(format_type),
+        );
+        match result {
+            Ok(camera) => Ok(camera),
+            Err(e) => bail!("create camera{} error:  {}", index, e),
+        }
+    }
+
+    pub fn get_camera_resolution(index: usize) -> ResultType<Resolution> {
+        let index = CameraIndex::Index(index as u32);
+        let camera = Self::create_camera(&index)?;
+        let resolution = camera.resolution();
+        Ok(Resolution {
+            width: resolution.width() as i32,
+            height: resolution.height() as i32,
+            ..Default::default()
+        })
+    }
+
+    pub fn get_sync_cameras() -> Vec<DisplayInfo> {
+        SYNC_CAMERA_DISPLAYS.lock().unwrap().clone()
+    }
+
+    pub fn get_capturer(current: usize) -> ResultType<Box<dyn TraitCapturer>> {
+        Ok(Box::new(CameraCapturer::new(current)?))
+    }
+}
+
+#[cfg(not(any(target_os = "windows", target_os = "linux")))]
+impl Cameras {
+    pub fn all_info() -> ResultType<Vec<DisplayInfo>> {
+        return Ok(Vec::new());
+    }
+
+    pub fn exists(_index: usize) -> bool {
+        false
+    }
+
+    pub fn get_camera_resolution(_index: usize) -> ResultType<Resolution> {
+        bail!(CAMERA_NOT_SUPPORTED);
+    }
+
+    pub fn get_sync_cameras() -> Vec<DisplayInfo> {
+        vec![]
+    }
+
+    pub fn get_capturer(_current: usize) -> ResultType<Box<dyn TraitCapturer>> {
+        bail!(CAMERA_NOT_SUPPORTED);
+    }
+}
+
+#[cfg(any(target_os = "windows", target_os = "linux"))]
+pub struct CameraCapturer {
+    camera: Camera,
+    data: Vec<u8>,
+    last_data: Vec<u8>, // for faster compare and copy
+}
+
+#[cfg(not(any(target_os = "windows", target_os = "linux")))]
+pub struct CameraCapturer;
+
+impl CameraCapturer {
+    #[cfg(any(target_os = "windows", target_os = "linux"))]
+    fn new(current: usize) -> ResultType<Self> {
+        let index = CameraIndex::Index(current as u32);
+        let camera = Cameras::create_camera(&index)?;
+        Ok(CameraCapturer {
+            camera,
+            data: Vec::new(),
+            last_data: Vec::new(),
+        })
+    }
+
+    #[allow(dead_code)]
+    #[cfg(not(any(target_os = "windows", target_os = "linux")))]
+    fn new(_current: usize) -> ResultType<Self> {
+        bail!(CAMERA_NOT_SUPPORTED);
+    }
+}
+
+impl TraitCapturer for CameraCapturer {
+    #[cfg(any(target_os = "windows", target_os = "linux"))]
+    fn frame<'a>(&'a mut self, _timeout: std::time::Duration) -> std::io::Result<Frame<'a>> {
+        // TODO: move this check outside `frame`.
+        if !self.camera.is_stream_open() {
+            if let Err(e) = self.camera.open_stream() {
+                return Err(io::Error::new(
+                    io::ErrorKind::Other,
+                    format!("Camera open stream error: {}", e),
+                ));
+            }
+        }
+        match self.camera.frame() {
+            Ok(buffer) => {
+                match buffer.decode_image::<RgbAFormat>() {
+                    Ok(decoded) => {
+                        self.data = decoded.as_raw().to_vec();
+                        crate::would_block_if_equal(&mut self.last_data, &self.data)?;
+                        // FIXME: macos's PixelBuffer cannot be directly created from bytes slice.
+                        cfg_if::cfg_if! {
+                            if #[cfg(any(target_os = "linux", target_os = "windows"))] {
+                                Ok(Frame::PixelBuffer(PixelBuffer::new(
+                                    &self.data,
+                                    Pixfmt::RGBA,
+                                    decoded.width() as usize,
+                                    decoded.height() as usize,
+                                )))
+                            } else {
+                                Err(io::Error::new(
+                                    io::ErrorKind::Other,
+                                    format!("Camera is not supported on this platform yet"),
+                                ))
+                            }
+                        }
+                    }
+                    Err(e) => Err(io::Error::new(
+                        io::ErrorKind::Other,
+                        format!("Camera frame decode error: {}", e),
+                    )),
+                }
+            }
+            Err(e) => Err(io::Error::new(
+                io::ErrorKind::Other,
+                format!("Camera frame error: {}", e),
+            )),
+        }
+    }
+
+    #[cfg(not(any(target_os = "windows", target_os = "linux")))]
+    fn frame<'a>(&'a mut self, _timeout: std::time::Duration) -> std::io::Result<Frame<'a>> {
+        Err(io::Error::new(
+            io::ErrorKind::Other,
+            CAMERA_NOT_SUPPORTED.to_string(),
+        ))
+    }
+
+    #[cfg(windows)]
+    fn is_gdi(&self) -> bool {
+        true
+    }
+
+    #[cfg(windows)]
+    fn set_gdi(&mut self) -> bool {
+        true
+    }
+
+    #[cfg(feature = "vram")]
+    fn device(&self) -> AdapterDevice {
+        AdapterDevice::default()
+    }
+
+    #[cfg(feature = "vram")]
+    fn set_output_texture(&mut self, _texture: bool) {}
+}

libs/scrap/src/common/convert.rs

@@ -0,0 +1,236 @@
+#![allow(non_camel_case_types)]
+#![allow(non_snake_case)]
+#![allow(non_upper_case_globals)]
+#![allow(improper_ctypes)]
+#![allow(dead_code)]
+
+include!(concat!(env!("OUT_DIR"), "/yuv_ffi.rs"));
+
+#[cfg(not(target_os = "ios"))]
+use crate::PixelBuffer;
+use crate::{generate_call_macro, EncodeYuvFormat, TraitPixelBuffer};
+use hbb_common::{bail, log, ResultType};
+
+generate_call_macro!(call_yuv, false);
+
+#[cfg(not(target_os = "ios"))]
+pub fn convert_to_yuv(
+    captured: &PixelBuffer,
+    dst_fmt: EncodeYuvFormat,
+    dst: &mut Vec<u8>,
+    mid_data: &mut Vec<u8>,
+) -> ResultType<()> {
+    let src = captured.data();
+    let src_stride = captured.stride();
+    let src_pixfmt = captured.pixfmt();
+    let src_width = captured.width();
+    let src_height = captured.height();
+    if src_width > dst_fmt.w || src_height > dst_fmt.h {
+        bail!(
+            "src rect > dst rect: ({src_width}, {src_height}) > ({},{})",
+            dst_fmt.w,
+            dst_fmt.h
+        );
+    }
+    if src_pixfmt == crate::Pixfmt::BGRA
+        || src_pixfmt == crate::Pixfmt::RGBA
+        || src_pixfmt == crate::Pixfmt::RGB565LE
+    {
+        // stride is calculated, not real, so we need to check it
+        if src_stride[0] < src_width * src_pixfmt.bytes_per_pixel() {
+            bail!(
+                "src_stride too small: {} < {}",
+                src_stride[0],
+                src_width * src_pixfmt.bytes_per_pixel()
+            );
+        }
+        if src.len() < src_stride[0] * src_height {
+            bail!(
+                "wrong src len, {} < {} * {}",
+                src.len(),
+                src_stride[0],
+                src_height
+            );
+        }
+    }
+    let align = |x: usize| (x + 63) / 64 * 64;
+    let unsupported = format!(
+        "unsupported pixfmt conversion: {src_pixfmt:?} -> {:?}",
+        dst_fmt.pixfmt
+    );
+
+    match (src_pixfmt, dst_fmt.pixfmt) {
+        (crate::Pixfmt::BGRA, crate::Pixfmt::I420)
+        | (crate::Pixfmt::RGBA, crate::Pixfmt::I420)
+        | (crate::Pixfmt::RGB565LE, crate::Pixfmt::I420) => {
+            let dst_stride_y = dst_fmt.stride[0];
+            let dst_stride_uv = dst_fmt.stride[1];
+            dst.resize(dst_fmt.h * dst_stride_y * 2, 0); // waste some memory to ensure memory safety
+            let dst_y = dst.as_mut_ptr();
+            let dst_u = dst[dst_fmt.u..].as_mut_ptr();
+            let dst_v = dst[dst_fmt.v..].as_mut_ptr();
+            let f = match src_pixfmt {
+                crate::Pixfmt::BGRA => ARGBToI420,
+                crate::Pixfmt::RGBA => ABGRToI420,
+                crate::Pixfmt::RGB565LE => RGB565ToI420,
+                _ => bail!(unsupported),
+            };
+            call_yuv!(f(
+                src.as_ptr(),
+                src_stride[0] as _,
+                dst_y,
+                dst_stride_y as _,
+                dst_u,
+                dst_stride_uv as _,
+                dst_v,
+                dst_stride_uv as _,
+                src_width as _,
+                src_height as _,
+            ));
+        }
+        (crate::Pixfmt::BGRA, crate::Pixfmt::NV12)
+        | (crate::Pixfmt::RGBA, crate::Pixfmt::NV12)
+        | (crate::Pixfmt::RGB565LE, crate::Pixfmt::NV12) => {
+            let dst_stride_y = dst_fmt.stride[0];
+            let dst_stride_uv = dst_fmt.stride[1];
+            dst.resize(
+                align(dst_fmt.h) * (align(dst_stride_y) + align(dst_stride_uv / 2)),
+                0,
+            );
+            let dst_y = dst.as_mut_ptr();
+            let dst_uv = dst[dst_fmt.u..].as_mut_ptr();
+            let (input, input_stride) = match src_pixfmt {
+                crate::Pixfmt::BGRA => (src.as_ptr(), src_stride[0]),
+                crate::Pixfmt::RGBA => (src.as_ptr(), src_stride[0]),
+                crate::Pixfmt::RGB565LE => {
+                    let mid_stride = src_width * 4;
+                    mid_data.resize(mid_stride * src_height, 0);
+                    call_yuv!(RGB565ToARGB(
+                        src.as_ptr(),
+                        src_stride[0] as _,
+                        mid_data.as_mut_ptr(),
+                        mid_stride as _,
+                        src_width as _,
+                        src_height as _,
+                    ));
+                    (mid_data.as_ptr(), mid_stride)
+                }
+                _ => bail!(unsupported),
+            };
+            let f = match src_pixfmt {
+                crate::Pixfmt::BGRA => ARGBToNV12,
+                crate::Pixfmt::RGBA => ABGRToNV12,
+                crate::Pixfmt::RGB565LE => ARGBToNV12,
+                _ => bail!(unsupported),
+            };
+            call_yuv!(f(
+                input,
+                input_stride as _,
+                dst_y,
+                dst_stride_y as _,
+                dst_uv,
+                dst_stride_uv as _,
+                src_width as _,
+                src_height as _,
+            ));
+        }
+        (crate::Pixfmt::BGRA, crate::Pixfmt::I444)
+        | (crate::Pixfmt::RGBA, crate::Pixfmt::I444)
+        | (crate::Pixfmt::RGB565LE, crate::Pixfmt::I444) => {
+            let dst_stride_y = dst_fmt.stride[0];
+            let dst_stride_u = dst_fmt.stride[1];
+            let dst_stride_v = dst_fmt.stride[2];
+            dst.resize(
+                align(dst_fmt.h)
+                    * (align(dst_stride_y) + align(dst_stride_u) + align(dst_stride_v)),
+                0,
+            );
+            let dst_y = dst.as_mut_ptr();
+            let dst_u = dst[dst_fmt.u..].as_mut_ptr();
+            let dst_v = dst[dst_fmt.v..].as_mut_ptr();
+            let (input, input_stride) = match src_pixfmt {
+                crate::Pixfmt::BGRA => (src.as_ptr(), src_stride[0]),
+                crate::Pixfmt::RGBA => {
+                    mid_data.resize(src.len(), 0);
+                    call_yuv!(ABGRToARGB(
+                        src.as_ptr(),
+                        src_stride[0] as _,
+                        mid_data.as_mut_ptr(),
+                        src_stride[0] as _,
+                        src_width as _,
+                        src_height as _,
+                    ));
+                    (mid_data.as_ptr(), src_stride[0])
+                }
+                crate::Pixfmt::RGB565LE => {
+                    let mid_stride = src_width * 4;
+                    mid_data.resize(mid_stride * src_height, 0);
+                    call_yuv!(RGB565ToARGB(
+                        src.as_ptr(),
+                        src_stride[0] as _,
+                        mid_data.as_mut_ptr(),
+                        mid_stride as _,
+                        src_width as _,
+                        src_height as _,
+                    ));
+                    (mid_data.as_ptr(), mid_stride)
+                }
+                _ => bail!(unsupported),
+            };
+
+            call_yuv!(ARGBToI444(
+                input,
+                input_stride as _,
+                dst_y,
+                dst_stride_y as _,
+                dst_u,
+                dst_stride_u as _,
+                dst_v,
+                dst_stride_v as _,
+                src_width as _,
+                src_height as _,
+            ));
+        }
+        _ => {
+            bail!(unsupported);
+        }
+    }
+    Ok(())
+}
+
+#[cfg(not(target_os = "ios"))]
+pub fn convert(captured: &PixelBuffer, pixfmt: crate::Pixfmt, dst: &mut Vec<u8>) -> ResultType<()> {
+    if captured.pixfmt() == pixfmt {
+        dst.extend_from_slice(captured.data());
+        return Ok(());
+    }
+
+    let src = captured.data();
+    let src_stride = captured.stride();
+    let src_pixfmt = captured.pixfmt();
+    let src_width = captured.width();
+    let src_height = captured.height();
+
+    let unsupported = format!(
+        "unsupported pixfmt conversion: {src_pixfmt:?} -> {:?}",
+        pixfmt
+    );
+
+    match (src_pixfmt, pixfmt) {
+        (crate::Pixfmt::BGRA, crate::Pixfmt::RGBA) | (crate::Pixfmt::RGBA, crate::Pixfmt::BGRA) => {
+            dst.resize(src.len(), 0);
+            call_yuv!(ABGRToARGB(
+                src.as_ptr(),
+                src_stride[0] as _,
+                dst.as_mut_ptr(),
+                src_stride[0] as _,
+                src_width as _,
+                src_height as _,
+            ));
+        }
+        _ => {
+            bail!(unsupported);
+        }
+    }
+    Ok(())
+}

libs/scrap/src/common/dxgi.rs

@@ -0,0 +1,264 @@
+#[cfg(feature = "vram")]
+use crate::AdapterDevice;
+use crate::{common::TraitCapturer, dxgi, Frame, Pixfmt};
+use std::{
+    io::{
+        self,
+        ErrorKind::{NotFound, TimedOut, WouldBlock},
+    },
+    time::Duration,
+};
+
+pub struct Capturer {
+    inner: dxgi::Capturer,
+    width: usize,
+    height: usize,
+}
+
+impl Capturer {
+    pub fn new(display: Display) -> io::Result<Capturer> {
+        let width = display.width();
+        let height = display.height();
+        let inner = dxgi::Capturer::new(display.0)?;
+        Ok(Capturer {
+            inner,
+            width,
+            height,
+        })
+    }
+
+    pub fn cancel_gdi(&mut self) {
+        self.inner.cancel_gdi()
+    }
+
+    pub fn width(&self) -> usize {
+        self.width
+    }
+
+    pub fn height(&self) -> usize {
+        self.height
+    }
+}
+
+impl TraitCapturer for Capturer {
+    fn frame<'a>(&'a mut self, timeout: Duration) -> io::Result<Frame<'a>> {
+        match self.inner.frame(timeout.as_millis() as _) {
+            Ok(frame) => Ok(frame),
+            Err(ref error) if error.kind() == TimedOut => Err(WouldBlock.into()),
+            Err(error) => Err(error),
+        }
+    }
+
+    fn is_gdi(&self) -> bool {
+        self.inner.is_gdi()
+    }
+
+    fn set_gdi(&mut self) -> bool {
+        self.inner.set_gdi()
+    }
+
+    #[cfg(feature = "vram")]
+    fn device(&self) -> AdapterDevice {
+        self.inner.device()
+    }
+
+    #[cfg(feature = "vram")]
+    fn set_output_texture(&mut self, texture: bool) {
+        self.inner.set_output_texture(texture);
+    }
+}
+
+pub struct PixelBuffer<'a> {
+    data: &'a [u8],
+    pixfmt: Pixfmt,
+    width: usize,
+    height: usize,
+    stride: Vec<usize>,
+}
+
+impl<'a> PixelBuffer<'a> {
+    pub fn new(data: &'a [u8], pixfmt: Pixfmt, width: usize, height: usize) -> Self {
+        let stride0 = data.len() / height;
+        let mut stride = Vec::new();
+        stride.push(stride0);
+        PixelBuffer {
+            data,
+            pixfmt,
+            width,
+            height,
+            stride,
+        }
+    }
+
+    #[allow(non_snake_case)]
+    pub fn with_BGRA(data: &'a [u8], width: usize, height: usize) -> Self {
+        Self::new(data, Pixfmt::BGRA, width, height)
+    }
+}
+
+impl<'a> crate::TraitPixelBuffer for PixelBuffer<'a> {
+    fn data(&self) -> &[u8] {
+        self.data
+    }
+
+    fn width(&self) -> usize {
+        self.width
+    }
+
+    fn height(&self) -> usize {
+        self.height
+    }
+
+    fn stride(&self) -> Vec<usize> {
+        self.stride.clone()
+    }
+
+    fn pixfmt(&self) -> Pixfmt {
+        self.pixfmt
+    }
+}
+
+pub struct Display(dxgi::Display);
+
+impl Display {
+    pub fn primary() -> io::Result<Display> {
+        // not implemented yet
+        Err(NotFound.into())
+    }
+
+    pub fn all() -> io::Result<Vec<Display>> {
+        let displays_gdi = dxgi::Displays::get_from_gdi()
+            .drain(..)
+            .map(Display)
+            .collect::<Vec<_>>();
+
+        let displays_dxgi = Self::all_().unwrap_or(Default::default());
+
+        // Return gdi displays if dxgi is not supported
+        if displays_dxgi.is_empty() {
+            println!("Display got from gdi");
+            return Ok(displays_gdi);
+        }
+
+        // Return dxgi displays if length is not equal
+        if displays_dxgi.len() != displays_gdi.len() {
+            return Ok(displays_dxgi);
+        }
+
+        // Check if names are equal
+        let names_gdi = displays_gdi.iter().map(|d| d.name()).collect::<Vec<_>>();
+        let names_dxgi = displays_dxgi.iter().map(|d| d.name()).collect::<Vec<_>>();
+        for name in names_gdi.iter() {
+            if !names_dxgi.contains(name) {
+                return Ok(displays_dxgi);
+            }
+        }
+
+        // Reorder displays from dxgi
+        let mut displays_dxgi = displays_dxgi;
+        let mut displays_dxgi_ordered = Vec::new();
+        for name in names_gdi.iter() {
+            let pos = match displays_dxgi.iter().position(|d| d.name() == *name) {
+                Some(pos) => pos,
+                None => {
+                    // unreachable!
+                    0
+                }
+            };
+            displays_dxgi_ordered.push(displays_dxgi.remove(pos));
+        }
+
+        Ok(displays_dxgi_ordered)
+    }
+
+    fn all_() -> io::Result<Vec<Display>> {
+        Ok(dxgi::Displays::new()?.map(Display).collect::<Vec<_>>())
+    }
+
+    pub fn width(&self) -> usize {
+        self.0.width() as usize
+    }
+
+    pub fn height(&self) -> usize {
+        self.0.height() as usize
+    }
+
+    pub fn name(&self) -> String {
+        use std::ffi::OsString;
+        use std::os::windows::prelude::*;
+        OsString::from_wide(self.0.name())
+            .to_string_lossy()
+            .to_string()
+    }
+
+    pub fn is_online(&self) -> bool {
+        self.0.is_online()
+    }
+
+    pub fn origin(&self) -> (i32, i32) {
+        self.0.origin()
+    }
+
+    pub fn is_primary(&self) -> bool {
+        // https://docs.microsoft.com/en-us/windows/win32/api/wingdi/ns-wingdi-devmodea
+        self.origin() == (0, 0)
+    }
+
+    #[cfg(feature = "vram")]
+    pub fn adapter_luid(&self) -> Option<i64> {
+        self.0.adapter_luid()
+    }
+}
+
+pub struct CapturerMag {
+    inner: dxgi::mag::CapturerMag,
+    data: Vec<u8>,
+}
+
+impl CapturerMag {
+    pub fn is_supported() -> bool {
+        dxgi::mag::CapturerMag::is_supported()
+    }
+
+    pub fn new(origin: (i32, i32), width: usize, height: usize) -> io::Result<Self> {
+        Ok(CapturerMag {
+            inner: dxgi::mag::CapturerMag::new(origin, width, height)?,
+            data: Vec::new(),
+        })
+    }
+
+    pub fn exclude(&mut self, cls: &str, name: &str) -> io::Result<bool> {
+        self.inner.exclude(cls, name)
+    }
+    // ((x, y), w, h)
+    pub fn get_rect(&self) -> ((i32, i32), usize, usize) {
+        self.inner.get_rect()
+    }
+}
+
+impl TraitCapturer for CapturerMag {
+    fn frame<'a>(&'a mut self, _timeout_ms: Duration) -> io::Result<Frame<'a>> {
+        self.inner.frame(&mut self.data)?;
+        Ok(Frame::PixelBuffer(PixelBuffer::with_BGRA(
+            &self.data,
+            self.inner.get_rect().1,
+            self.inner.get_rect().2,
+        )))
+    }
+
+    fn is_gdi(&self) -> bool {
+        false
+    }
+
+    fn set_gdi(&mut self) -> bool {
+        false
+    }
+
+    #[cfg(feature = "vram")]
+    fn device(&self) -> AdapterDevice {
+        AdapterDevice::default()
+    }
+
+    #[cfg(feature = "vram")]
+    fn set_output_texture(&mut self, _texture: bool) {}
+}

libs/scrap/src/common/hwcodec.rs

@@ -0,0 +1,763 @@
+use crate::{
+    codec::{base_bitrate, codec_thread_num, enable_hwcodec_option, EncoderApi, EncoderCfg},
+    convert::*,
+    CodecFormat, EncodeInput, ImageFormat, ImageRgb, Pixfmt, HW_STRIDE_ALIGN,
+};
+use hbb_common::{
+    anyhow::{anyhow, bail, Context},
+    bytes::Bytes,
+    log,
+    message_proto::{EncodedVideoFrame, EncodedVideoFrames, VideoFrame},
+    serde_derive::{Deserialize, Serialize},
+    serde_json, ResultType,
+};
+use hwcodec::{
+    common::{
+        DataFormat, HwcodecErrno,
+        Quality::{self, *},
+        RateControl::{self, *},
+    },
+    ffmpeg::AVPixelFormat,
+    ffmpeg_ram::{
+        decode::{DecodeContext, DecodeFrame, Decoder},
+        encode::{EncodeContext, EncodeFrame, Encoder},
+        ffmpeg_linesize_offset_length, CodecInfo,
+    },
+};
+
+const DEFAULT_PIXFMT: AVPixelFormat = AVPixelFormat::AV_PIX_FMT_NV12;
+pub const DEFAULT_FPS: i32 = 30;
+const DEFAULT_GOP: i32 = i32::MAX;
+const DEFAULT_HW_QUALITY: Quality = Quality_Default;
+pub const ERR_HEVC_POC: i32 = HwcodecErrno::HWCODEC_ERR_HEVC_COULD_NOT_FIND_POC as i32;
+
+crate::generate_call_macro!(call_yuv, false);
+
+#[cfg(not(target_os = "android"))]
+lazy_static::lazy_static! {
+    static ref CONFIG: std::sync::Arc<std::sync::Mutex<Option<HwCodecConfig>>> = Default::default();
+    static ref CONFIG_SET_BY_IPC: std::sync::Arc<std::sync::Mutex<bool>> = Default::default();
+}
+
+#[derive(Debug, Clone)]
+pub struct HwRamEncoderConfig {
+    pub name: String,
+    pub mc_name: Option<String>,
+    pub width: usize,
+    pub height: usize,
+    pub quality: f32,
+    pub keyframe_interval: Option<usize>,
+}
+
+pub struct HwRamEncoder {
+    encoder: Encoder,
+    pub format: DataFormat,
+    pub pixfmt: AVPixelFormat,
+    bitrate: u32, //kbs
+    config: HwRamEncoderConfig,
+}
+
+impl EncoderApi for HwRamEncoder {
+    fn new(cfg: EncoderCfg, _i444: bool) -> ResultType<Self>
+    where
+        Self: Sized,
+    {
+        match cfg {
+            EncoderCfg::HWRAM(config) => {
+                let rc = Self::rate_control(&config);
+                let mut bitrate =
+                    Self::bitrate(&config.name, config.width, config.height, config.quality);
+                bitrate = Self::check_bitrate_range(&config, bitrate);
+                let gop = config.keyframe_interval.unwrap_or(DEFAULT_GOP as _) as i32;
+                let ctx = EncodeContext {
+                    name: config.name.clone(),
+                    mc_name: config.mc_name.clone(),
+                    width: config.width as _,
+                    height: config.height as _,
+                    pixfmt: DEFAULT_PIXFMT,
+                    align: HW_STRIDE_ALIGN as _,
+                    kbs: bitrate as i32,
+                    fps: DEFAULT_FPS,
+                    gop,
+                    quality: DEFAULT_HW_QUALITY,
+                    rc,
+                    q: -1,
+                    thread_count: codec_thread_num(16) as _, // ffmpeg's thread_count is used for cpu
+                };
+                let format = match Encoder::format_from_name(config.name.clone()) {
+                    Ok(format) => format,
+                    Err(_) => {
+                        return Err(anyhow!(format!(
+                            "failed to get format from name:{}",
+                            config.name
+                        )))
+                    }
+                };
+                match Encoder::new(ctx.clone()) {
+                    Ok(encoder) => Ok(HwRamEncoder {
+                        encoder,
+                        format,
+                        pixfmt: ctx.pixfmt,
+                        bitrate,
+                        config,
+                    }),
+                    Err(_) => Err(anyhow!(format!("Failed to create encoder"))),
+                }
+            }
+            _ => Err(anyhow!("encoder type mismatch")),
+        }
+    }
+
+    fn encode_to_message(&mut self, input: EncodeInput, ms: i64) -> ResultType<VideoFrame> {
+        let mut vf = VideoFrame::new();
+        let mut frames = Vec::new();
+        for frame in self
+            .encode(input.yuv()?, ms)
+            .with_context(|| "Failed to encode")?
+        {
+            frames.push(EncodedVideoFrame {
+                data: Bytes::from(frame.data),
+                pts: frame.pts,
+                key: frame.key == 1,
+                ..Default::default()
+            });
+        }
+        if frames.len() > 0 {
+            let frames = EncodedVideoFrames {
+                frames: frames.into(),
+                ..Default::default()
+            };
+            match self.format {
+                DataFormat::H264 => vf.set_h264s(frames),
+                DataFormat::H265 => vf.set_h265s(frames),
+                _ => bail!("unsupported format: {:?}", self.format),
+            }
+            Ok(vf)
+        } else {
+            Err(anyhow!("no valid frame"))
+        }
+    }
+
+    fn yuvfmt(&self) -> crate::EncodeYuvFormat {
+        let pixfmt = if self.pixfmt == AVPixelFormat::AV_PIX_FMT_NV12 {
+            Pixfmt::NV12
+        } else {
+            Pixfmt::I420
+        };
+        let stride = self
+            .encoder
+            .linesize
+            .clone()
+            .drain(..)
+            .map(|i| i as usize)
+            .collect();
+        crate::EncodeYuvFormat {
+            pixfmt,
+            w: self.encoder.ctx.width as _,
+            h: self.encoder.ctx.height as _,
+            stride,
+            u: self.encoder.offset[0] as _,
+            v: if pixfmt == Pixfmt::NV12 {
+                0
+            } else {
+                self.encoder.offset[1] as _
+            },
+        }
+    }
+
+    #[cfg(feature = "vram")]
+    fn input_texture(&self) -> bool {
+        false
+    }
+
+    fn set_quality(&mut self, ratio: f32) -> ResultType<()> {
+        let mut bitrate = Self::bitrate(
+            &self.config.name,
+            self.config.width,
+            self.config.height,
+            ratio,
+        );
+        if bitrate > 0 {
+            bitrate = Self::check_bitrate_range(&self.config, bitrate);
+            self.encoder.set_bitrate(bitrate as _).ok();
+            self.bitrate = bitrate;
+        }
+        self.config.quality = ratio;
+        Ok(())
+    }
+
+    fn bitrate(&self) -> u32 {
+        self.bitrate
+    }
+
+    fn support_changing_quality(&self) -> bool {
+        ["vaapi"].iter().all(|&x| !self.config.name.contains(x))
+    }
+
+    fn latency_free(&self) -> bool {
+        ["mediacodec", "videotoolbox"]
+            .iter()
+            .all(|&x| !self.config.name.contains(x))
+    }
+
+    fn is_hardware(&self) -> bool {
+        true
+    }
+
+    fn disable(&self) {
+        HwCodecConfig::clear(false, true);
+    }
+}
+
+impl HwRamEncoder {
+    pub fn try_get(format: CodecFormat) -> Option<CodecInfo> {
+        let mut info = None;
+        let best = CodecInfo::prioritized(HwCodecConfig::get().ram_encode);
+        match format {
+            CodecFormat::H264 => {
+                if let Some(v) = best.h264 {
+                    info = Some(v);
+                }
+            }
+            CodecFormat::H265 => {
+                if let Some(v) = best.h265 {
+                    info = Some(v);
+                }
+            }
+            _ => {}
+        }
+        info
+    }
+
+    pub fn encode(&mut self, yuv: &[u8], ms: i64) -> ResultType<Vec<EncodeFrame>> {
+        match self.encoder.encode(yuv, ms) {
+            Ok(v) => {
+                let mut data = Vec::<EncodeFrame>::new();
+                data.append(v);
+                Ok(data)
+            }
+            Err(_) => Ok(Vec::<EncodeFrame>::new()),
+        }
+    }
+
+    fn rate_control(_config: &HwRamEncoderConfig) -> RateControl {
+        #[cfg(target_os = "android")]
+        if _config.name.contains("mediacodec") {
+            return RC_VBR;
+        }
+        RC_CBR
+    }
+
+    pub fn bitrate(name: &str, width: usize, height: usize, ratio: f32) -> u32 {
+        Self::calc_bitrate(width, height, ratio, name.contains("h264"))
+    }
+
+    pub fn calc_bitrate(width: usize, height: usize, ratio: f32, h264: bool) -> u32 {
+        let base = base_bitrate(width as _, height as _) as f32 * ratio;
+        let threshold = 2000.0;
+        let decay_rate = 0.001; // 1000 * 0.001 = 1
+        let factor: f32 = if cfg!(target_os = "android") {
+            // https://stackoverflow.com/questions/26110337/what-are-valid-bit-rates-to-set-for-mediacodec?rq=3
+            if base > threshold {
+                1.0 + 4.0 / (1.0 + (base - threshold) * decay_rate)
+            } else {
+                5.0
+            }
+        } else if h264 {
+            if base > threshold {
+                1.0 + 1.0 / (1.0 + (base - threshold) * decay_rate)
+            } else {
+                2.0
+            }
+        } else {
+            if base > threshold {
+                1.0 + 0.5 / (1.0 + (base - threshold) * decay_rate)
+            } else {
+                1.5
+            }
+        };
+        (base * factor) as u32
+    }
+
+    pub fn check_bitrate_range(_config: &HwRamEncoderConfig, bitrate: u32) -> u32 {
+        #[cfg(target_os = "android")]
+        if _config.name.contains("mediacodec") {
+            let info = crate::android::ffi::get_codec_info();
+            if let Some(info) = info {
+                if let Some(codec) = info
+                    .codecs
+                    .iter()
+                    .find(|c| Some(c.name.clone()) == _config.mc_name && c.is_encoder)
+                {
+                    if codec.max_bitrate > codec.min_bitrate {
+                        if bitrate > codec.max_bitrate {
+                            return codec.max_bitrate;
+                        }
+                        if bitrate < codec.min_bitrate {
+                            return codec.min_bitrate;
+                        }
+                    }
+                }
+            }
+        }
+        bitrate
+    }
+}
+
+pub struct HwRamDecoder {
+    decoder: Decoder,
+    pub info: CodecInfo,
+}
+
+impl HwRamDecoder {
+    pub fn try_get(format: CodecFormat) -> Option<CodecInfo> {
+        let mut info = None;
+        let soft = CodecInfo::soft();
+        match format {
+            CodecFormat::H264 => {
+                if let Some(v) = soft.h264 {
+                    info = Some(v);
+                }
+            }
+            CodecFormat::H265 => {
+                if let Some(v) = soft.h265 {
+                    info = Some(v);
+                }
+            }
+            _ => {}
+        }
+        if enable_hwcodec_option() {
+            let best = CodecInfo::prioritized(HwCodecConfig::get().ram_decode);
+            match format {
+                CodecFormat::H264 => {
+                    if let Some(v) = best.h264 {
+                        info = Some(v);
+                    }
+                }
+                CodecFormat::H265 => {
+                    if let Some(v) = best.h265 {
+                        info = Some(v);
+                    }
+                }
+                _ => {}
+            }
+        }
+        info
+    }
+
+    pub fn new(format: CodecFormat) -> ResultType<Self> {
+        let info = HwRamDecoder::try_get(format);
+        log::info!("try create {info:?} ram decoder");
+        let Some(info) = info else {
+            bail!("unsupported format: {:?}", format);
+        };
+        let ctx = DecodeContext {
+            name: info.name.clone(),
+            device_type: info.hwdevice.clone(),
+            thread_count: codec_thread_num(16) as _,
+        };
+        match Decoder::new(ctx) {
+            Ok(decoder) => Ok(HwRamDecoder { decoder, info }),
+            Err(_) => {
+                HwCodecConfig::clear(false, false);
+                Err(anyhow!(format!("Failed to create decoder")))
+            }
+        }
+    }
+    pub fn decode<'a>(&'a mut self, data: &[u8]) -> ResultType<Vec<HwRamDecoderImage<'a>>> {
+        match self.decoder.decode(data) {
+            Ok(v) => Ok(v.iter().map(|f| HwRamDecoderImage { frame: f }).collect()),
+            Err(e) => Err(anyhow!(e)),
+        }
+    }
+}
+
+pub struct HwRamDecoderImage<'a> {
+    frame: &'a DecodeFrame,
+}
+
+impl HwRamDecoderImage<'_> {
+    // rgb [in/out] fmt and stride must be set in ImageRgb
+    pub fn to_fmt(&self, rgb: &mut ImageRgb, i420: &mut Vec<u8>) -> ResultType<()> {
+        let frame = self.frame;
+        let width = frame.width;
+        let height = frame.height;
+        rgb.w = width as _;
+        rgb.h = height as _;
+        let dst_align = rgb.align();
+        let bytes_per_row = (rgb.w * 4 + dst_align - 1) & !(dst_align - 1);
+        rgb.raw.resize(rgb.h * bytes_per_row, 0);
+        match frame.pixfmt {
+            AVPixelFormat::AV_PIX_FMT_NV12 => {
+                // I420ToARGB is much faster than NV12ToARGB in tests on Windows
+                if cfg!(windows) {
+                    let Ok((linesize_i420, offset_i420, len_i420)) = ffmpeg_linesize_offset_length(
+                        AVPixelFormat::AV_PIX_FMT_YUV420P,
+                        width as _,
+                        height as _,
+                        HW_STRIDE_ALIGN,
+                    ) else {
+                        bail!("failed to get i420 linesize, offset, length");
+                    };
+                    i420.resize(len_i420 as _, 0);
+                    let i420_offset_y = unsafe { i420.as_ptr().add(0) as _ };
+                    let i420_offset_u = unsafe { i420.as_ptr().add(offset_i420[0] as _) as _ };
+                    let i420_offset_v = unsafe { i420.as_ptr().add(offset_i420[1] as _) as _ };
+                    call_yuv!(NV12ToI420(
+                        frame.data[0].as_ptr(),
+                        frame.linesize[0],
+                        frame.data[1].as_ptr(),
+                        frame.linesize[1],
+                        i420_offset_y,
+                        linesize_i420[0],
+                        i420_offset_u,
+                        linesize_i420[1],
+                        i420_offset_v,
+                        linesize_i420[2],
+                        width,
+                        height,
+                    ));
+                    let f = match rgb.fmt() {
+                        ImageFormat::ARGB => I420ToARGB,
+                        ImageFormat::ABGR => I420ToABGR,
+                        _ => bail!("unsupported format: {:?} -> {:?}", frame.pixfmt, rgb.fmt()),
+                    };
+                    call_yuv!(f(
+                        i420_offset_y,
+                        linesize_i420[0],
+                        i420_offset_u,
+                        linesize_i420[1],
+                        i420_offset_v,
+                        linesize_i420[2],
+                        rgb.raw.as_mut_ptr(),
+                        bytes_per_row as _,
+                        width,
+                        height,
+                    ));
+                } else {
+                    let f = match rgb.fmt() {
+                        ImageFormat::ARGB => NV12ToARGB,
+                        ImageFormat::ABGR => NV12ToABGR,
+                        _ => bail!("unsupported format: {:?} -> {:?}", frame.pixfmt, rgb.fmt()),
+                    };
+                    call_yuv!(f(
+                        frame.data[0].as_ptr(),
+                        frame.linesize[0],
+                        frame.data[1].as_ptr(),
+                        frame.linesize[1],
+                        rgb.raw.as_mut_ptr(),
+                        bytes_per_row as _,
+                        width,
+                        height,
+                    ));
+                }
+            }
+            AVPixelFormat::AV_PIX_FMT_YUV420P => {
+                let f = match rgb.fmt() {
+                    ImageFormat::ARGB => I420ToARGB,
+                    ImageFormat::ABGR => I420ToABGR,
+                    _ => bail!("unsupported format: {:?} -> {:?}", frame.pixfmt, rgb.fmt()),
+                };
+                call_yuv!(f(
+                    frame.data[0].as_ptr(),
+                    frame.linesize[0],
+                    frame.data[1].as_ptr(),
+                    frame.linesize[1],
+                    frame.data[2].as_ptr(),
+                    frame.linesize[2],
+                    rgb.raw.as_mut_ptr(),
+                    bytes_per_row as _,
+                    width,
+                    height,
+                ));
+            }
+        }
+        Ok(())
+    }
+}
+
+#[cfg(target_os = "android")]
+fn get_mime_type(codec: DataFormat) -> &'static str {
+    match codec {
+        DataFormat::VP8 => "video/x-vnd.on2.vp8",
+        DataFormat::VP9 => "video/x-vnd.on2.vp9",
+        DataFormat::AV1 => "video/av01",
+        DataFormat::H264 => "video/avc",
+        DataFormat::H265 => "video/hevc",
+    }
+}
+
+#[derive(Debug, Default, Serialize, Deserialize, Clone)]
+pub struct HwCodecConfig {
+    #[serde(default)]
+    pub signature: u64,
+    #[serde(default)]
+    pub ram_encode: Vec<CodecInfo>,
+    #[serde(default)]
+    pub ram_decode: Vec<CodecInfo>,
+    #[cfg(feature = "vram")]
+    #[serde(default)]
+    pub vram_encode: Vec<hwcodec::vram::FeatureContext>,
+    #[cfg(feature = "vram")]
+    #[serde(default)]
+    pub vram_decode: Vec<hwcodec::vram::DecodeContext>,
+}
+
+// HwCodecConfig2 is used to store the config in json format,
+// confy can't serde HwCodecConfig successfully if the non-first struct Vec is empty due to old toml version.
+// struct T { a: Vec<A>, b: Vec<String>} will fail if b is empty, but struct T { a: Vec<String>, b: Vec<String>} is ok.
+#[derive(Debug, Default, Serialize, Deserialize, Clone)]
+struct HwCodecConfig2 {
+    #[serde(default)]
+    pub config: String,
+}
+
+// ipc server process start check process once, other process get from ipc server once
+// install: --server start check process, check process send to --server,  ui get from --server
+// portable: ui start check process, check process send to ui
+// sciter and unilink: get from ipc server
+impl HwCodecConfig {
+    #[cfg(not(any(target_os = "android", target_os = "ios")))]
+    pub fn set(config: String) {
+        let config = serde_json::from_str(&config).unwrap_or_default();
+        log::info!("set hwcodec config");
+        log::debug!("{config:?}");
+        #[cfg(any(windows, target_os = "macos"))]
+        hbb_common::config::common_store(
+            &HwCodecConfig2 {
+                config: serde_json::to_string_pretty(&config).unwrap_or_default(),
+            },
+            "_hwcodec",
+        );
+        *CONFIG.lock().unwrap() = Some(config);
+        *CONFIG_SET_BY_IPC.lock().unwrap() = true;
+    }
+
+    pub fn get() -> HwCodecConfig {
+        #[cfg(target_os = "android")]
+        {
+            let info = crate::android::ffi::get_codec_info();
+            log::info!("all codec info: {info:?}");
+            struct T {
+                name_prefix: &'static str,
+                data_format: DataFormat,
+            }
+            let ts = vec![
+                T {
+                    name_prefix: "h264",
+                    data_format: DataFormat::H264,
+                },
+                T {
+                    name_prefix: "hevc",
+                    data_format: DataFormat::H265,
+                },
+            ];
+            let mut e = vec![];
+            if let Some(info) = info {
+                ts.iter().for_each(|t| {
+                    let codecs: Vec<_> = info
+                        .codecs
+                        .iter()
+                        .filter(|c| {
+                            c.is_encoder
+                                && c.mime_type.as_str() == get_mime_type(t.data_format)
+                                && c.nv12
+                                && c.hw == Some(true) //only use hardware codec
+                        })
+                        .collect();
+                    let screen_wh = std::cmp::max(info.w, info.h);
+                    let mut best = None;
+                    if let Some(codec) = codecs
+                        .iter()
+                        .find(|c| c.max_width >= screen_wh && c.max_height >= screen_wh)
+                    {
+                        best = Some(codec.name.clone());
+                    } else {
+                        // find the max resolution
+                        let mut max_area = 0;
+                        for codec in codecs.iter() {
+                            if codec.max_width * codec.max_height > max_area {
+                                best = Some(codec.name.clone());
+                                max_area = codec.max_width * codec.max_height;
+                            }
+                        }
+                    }
+                    if let Some(best) = best {
+                        e.push(CodecInfo {
+                            name: format!("{}_mediacodec", t.name_prefix),
+                            mc_name: Some(best),
+                            format: t.data_format,
+                            hwdevice: hwcodec::ffmpeg::AVHWDeviceType::AV_HWDEVICE_TYPE_NONE,
+                            priority: 0,
+                        });
+                    }
+                });
+            }
+            log::debug!("e: {e:?}");
+            HwCodecConfig {
+                ram_encode: e,
+                ..Default::default()
+            }
+        }
+        #[cfg(any(windows, target_os = "macos"))]
+        {
+            let config = CONFIG.lock().unwrap().clone();
+            match config {
+                Some(c) => c,
+                None => {
+                    log::info!("try load cached hwcodec config");
+                    let c = hbb_common::config::common_load::<HwCodecConfig2>("_hwcodec");
+                    let c: HwCodecConfig = serde_json::from_str(&c.config).unwrap_or_default();
+                    let new_signature = hwcodec::common::get_gpu_signature();
+                    if c.signature == new_signature {
+                        log::debug!("load cached hwcodec config: {c:?}");
+                        *CONFIG.lock().unwrap() = Some(c.clone());
+                        c
+                    } else {
+                        log::info!(
+                            "gpu signature changed, {} -> {}",
+                            c.signature,
+                            new_signature
+                        );
+                        HwCodecConfig::default()
+                    }
+                }
+            }
+        }
+        #[cfg(target_os = "linux")]
+        {
+            CONFIG.lock().unwrap().clone().unwrap_or_default()
+        }
+        #[cfg(target_os = "ios")]
+        {
+            HwCodecConfig::default()
+        }
+    }
+
+    #[cfg(not(any(target_os = "android", target_os = "ios")))]
+    pub fn get_set_value() -> Option<HwCodecConfig> {
+        let set = CONFIG_SET_BY_IPC.lock().unwrap().clone();
+        if set {
+            CONFIG.lock().unwrap().clone()
+        } else {
+            None
+        }
+    }
+
+    #[cfg(not(any(target_os = "android", target_os = "ios")))]
+    pub fn already_set() -> bool {
+        CONFIG_SET_BY_IPC.lock().unwrap().clone()
+    }
+
+    pub fn clear(vram: bool, encode: bool) {
+        log::info!("clear hwcodec config, vram: {vram}, encode: {encode}");
+        #[cfg(target_os = "android")]
+        crate::android::ffi::clear_codec_info();
+        #[cfg(not(target_os = "android"))]
+        {
+            let mut c = CONFIG.lock().unwrap();
+            if let Some(c) = c.as_mut() {
+                if vram {
+                    #[cfg(feature = "vram")]
+                    if encode {
+                        c.vram_encode = vec![];
+                    } else {
+                        c.vram_decode = vec![];
+                    }
+                } else {
+                    if encode {
+                        c.ram_encode = vec![];
+                    } else {
+                        c.ram_decode = vec![];
+                    }
+                }
+            }
+        }
+        crate::codec::Encoder::update(crate::codec::EncodingUpdate::Check);
+    }
+}
+
+pub fn check_available_hwcodec() -> String {
+    #[cfg(any(target_os = "linux", target_os = "macos"))]
+    hwcodec::common::setup_parent_death_signal();
+    let ctx = EncodeContext {
+        name: String::from(""),
+        mc_name: None,
+        width: 1280,
+        height: 720,
+        pixfmt: DEFAULT_PIXFMT,
+        align: HW_STRIDE_ALIGN as _,
+        kbs: 1000,
+        fps: DEFAULT_FPS,
+        gop: DEFAULT_GOP,
+        quality: DEFAULT_HW_QUALITY,
+        rc: RC_CBR,
+        q: -1,
+        thread_count: 4,
+    };
+    #[cfg(feature = "vram")]
+    let vram = crate::vram::check_available_vram();
+    #[cfg(feature = "vram")]
+    let vram_string = vram.2;
+    #[cfg(not(feature = "vram"))]
+    let vram_string = "".to_owned();
+    let c = HwCodecConfig {
+        ram_encode: Encoder::available_encoders(ctx, Some(vram_string)),
+        ram_decode: Decoder::available_decoders(),
+        #[cfg(feature = "vram")]
+        vram_encode: vram.0,
+        #[cfg(feature = "vram")]
+        vram_decode: vram.1,
+        signature: hwcodec::common::get_gpu_signature(),
+    };
+    log::debug!("{c:?}");
+    serde_json::to_string(&c).unwrap_or_default()
+}
+
+#[cfg(not(any(target_os = "android", target_os = "ios")))]
+pub fn start_check_process() {
+    if !enable_hwcodec_option() || HwCodecConfig::already_set() {
+        return;
+    }
+    use hbb_common::allow_err;
+    use std::sync::Once;
+    let f = || {
+        if let Ok(exe) = std::env::current_exe() {
+            if let Some(_) = exe.file_name().to_owned() {
+                let arg = "--check-hwcodec-config";
+                if let Ok(mut child) = std::process::Command::new(exe).arg(arg).spawn() {
+                    #[cfg(windows)]
+                    hwcodec::common::child_exit_when_parent_exit(child.id());
+                    // wait up to 30 seconds, it maybe slow on windows startup for poorly performing machines
+                    for _ in 0..30 {
+                        std::thread::sleep(std::time::Duration::from_secs(1));
+                        if let Ok(Some(_)) = child.try_wait() {
+                            break;
+                        }
+                    }
+                    allow_err!(child.kill());
+                    std::thread::sleep(std::time::Duration::from_millis(30));
+                    match child.try_wait() {
+                        Ok(Some(status)) => {
+                            log::info!("Check hwcodec config, exit with: {status}")
+                        }
+                        Ok(None) => {
+                            log::info!(
+                                "Check hwcodec config, status not ready yet, let's really wait"
+                            );
+                            let res = child.wait();
+                            log::info!("Check hwcodec config, wait result: {res:?}");
+                        }
+                        Err(e) => {
+                            log::error!("Check hwcodec config, error attempting to wait: {e}")
+                        }
+                    }
+                }
+            }
+        };
+    };
+    static ONCE: Once = Once::new();
+    ONCE.call_once(|| {
+        std::thread::spawn(f);
+    });
+}

libs/scrap/src/common/linux.rs

@@ -0,0 +1,139 @@
+use crate::{
+    common::{
+        wayland,
+        x11::{self},
+        TraitCapturer,
+    },
+    Frame,
+};
+use std::{io, time::Duration};
+
+pub enum Capturer {
+    X11(x11::Capturer),
+    WAYLAND(wayland::Capturer),
+}
+
+impl Capturer {
+    pub fn new(display: Display) -> io::Result<Capturer> {
+        Ok(match display {
+            Display::X11(d) => Capturer::X11(x11::Capturer::new(d)?),
+            Display::WAYLAND(d) => Capturer::WAYLAND(wayland::Capturer::new(d)?),
+        })
+    }
+
+    pub fn width(&self) -> usize {
+        match self {
+            Capturer::X11(d) => d.width(),
+            Capturer::WAYLAND(d) => d.width(),
+        }
+    }
+
+    pub fn height(&self) -> usize {
+        match self {
+            Capturer::X11(d) => d.height(),
+            Capturer::WAYLAND(d) => d.height(),
+        }
+    }
+}
+
+impl TraitCapturer for Capturer {
+    fn frame<'a>(&'a mut self, timeout: Duration) -> io::Result<Frame<'a>> {
+        match self {
+            Capturer::X11(d) => d.frame(timeout),
+            Capturer::WAYLAND(d) => d.frame(timeout),
+        }
+    }
+}
+
+pub enum Display {
+    X11(x11::Display),
+    WAYLAND(wayland::Display),
+}
+
+impl Display {
+    pub fn primary() -> io::Result<Display> {
+        Ok(if super::is_x11() {
+            Display::X11(x11::Display::primary()?)
+        } else {
+            Display::WAYLAND(wayland::Display::primary()?)
+        })
+    }
+
+    // Currently, wayland need to call wayland::clear() before call Display::all()
+    pub fn all() -> io::Result<Vec<Display>> {
+        Ok(if super::is_x11() {
+            x11::Display::all()?
+                .drain(..)
+                .map(|x| Display::X11(x))
+                .collect()
+        } else {
+            wayland::Display::all()?
+                .drain(..)
+                .map(|x| Display::WAYLAND(x))
+                .collect()
+        })
+    }
+
+    pub fn width(&self) -> usize {
+        match self {
+            Display::X11(d) => d.width(),
+            Display::WAYLAND(d) => d.width(),
+        }
+    }
+
+    pub fn height(&self) -> usize {
+        match self {
+            Display::X11(d) => d.height(),
+            Display::WAYLAND(d) => d.height(),
+        }
+    }
+
+    pub fn scale(&self) -> f64 {
+        match self {
+            Display::X11(_d) => 1.0,
+            Display::WAYLAND(d) => d.scale(),
+        }
+    }
+
+    pub fn logical_width(&self) -> usize {
+        match self {
+            Display::X11(d) => d.width(),
+            Display::WAYLAND(d) => d.logical_width(),
+        }
+    }
+
+    pub fn logical_height(&self) -> usize {
+        match self {
+            Display::X11(d) => d.height(),
+            Display::WAYLAND(d) => d.logical_height(),
+        }
+    }
+
+    pub fn origin(&self) -> (i32, i32) {
+        match self {
+            Display::X11(d) => d.origin(),
+            Display::WAYLAND(d) => d.origin(),
+        }
+    }
+
+    pub fn is_online(&self) -> bool {
+        match self {
+            Display::X11(d) => d.is_online(),
+            Display::WAYLAND(d) => d.is_online(),
+        }
+    }
+
+    pub fn is_primary(&self) -> bool {
+        match self {
+            Display::X11(d) => d.is_primary(),
+            Display::WAYLAND(d) => d.is_primary(),
+        }
+    }
+
+    pub fn name(&self) -> String {
+        match self {
+            Display::X11(d) => d.name(),
+            Display::WAYLAND(d) => d.name(),
+        }
+    }
+}

libs/scrap/src/common/mediacodec.rs

@@ -0,0 +1,171 @@
+use hbb_common::{anyhow::Error, bail, log, ResultType};
+use ndk::media::media_codec::{MediaCodec, MediaCodecDirection, MediaFormat};
+use std::ops::Deref;
+use std::{
+    io::Write,
+    sync::atomic::{AtomicBool, Ordering},
+    time::Duration,
+};
+
+use crate::ImageFormat;
+use crate::{
+    codec::{EncoderApi, EncoderCfg},
+    CodecFormat, I420ToABGR, I420ToARGB, ImageRgb,
+};
+
+/// MediaCodec mime type name
+const H264_MIME_TYPE: &str = "video/avc";
+const H265_MIME_TYPE: &str = "video/hevc";
+// const VP8_MIME_TYPE: &str = "video/x-vnd.on2.vp8";
+// const VP9_MIME_TYPE: &str = "video/x-vnd.on2.vp9";
+
+// TODO MediaCodecEncoder
+
+pub static H264_DECODER_SUPPORT: AtomicBool = AtomicBool::new(false);
+pub static H265_DECODER_SUPPORT: AtomicBool = AtomicBool::new(false);
+
+pub struct MediaCodecDecoder {
+    decoder: MediaCodec,
+    name: String,
+}
+
+impl Deref for MediaCodecDecoder {
+    type Target = MediaCodec;
+
+    fn deref(&self) -> &Self::Target {
+        &self.decoder
+    }
+}
+
+impl MediaCodecDecoder {
+    pub fn new(format: CodecFormat) -> Option<MediaCodecDecoder> {
+        match format {
+            CodecFormat::H264 => create_media_codec(H264_MIME_TYPE, MediaCodecDirection::Decoder),
+            CodecFormat::H265 => create_media_codec(H265_MIME_TYPE, MediaCodecDirection::Decoder),
+            _ => {
+                log::error!("Unsupported codec format: {}", format);
+                None
+            }
+        }
+    }
+
+    // rgb [in/out] fmt and stride must be set in ImageRgb
+    pub fn decode(&mut self, data: &[u8], rgb: &mut ImageRgb) -> ResultType<bool> {
+        // take dst_stride into account please
+        let dst_stride = rgb.stride();
+        match self.dequeue_input_buffer(Duration::from_millis(10))? {
+            Some(mut input_buffer) => {
+                let mut buf = input_buffer.buffer_mut();
+                if data.len() > buf.len() {
+                    log::error!("Failed to decode, the input data size is bigger than input buf");
+                    bail!("The input data size is bigger than input buf");
+                }
+                buf.write_all(&data)?;
+                self.queue_input_buffer(input_buffer, 0, data.len(), 0, 0)?;
+            }
+            None => {
+                log::debug!("Failed to dequeue_input_buffer: No available input_buffer");
+            }
+        };
+
+        return match self.dequeue_output_buffer(Duration::from_millis(100))? {
+            Some(output_buffer) => {
+                let res_format = self.output_format();
+                let w = res_format
+                    .i32("width")
+                    .ok_or(Error::msg("Failed to dequeue_output_buffer, width is None"))?
+                    as usize;
+                let h = res_format.i32("height").ok_or(Error::msg(
+                    "Failed to dequeue_output_buffer, height is None",
+                ))? as usize;
+                let stride = res_format.i32("stride").ok_or(Error::msg(
+                    "Failed to dequeue_output_buffer, stride is None",
+                ))?;
+                let buf = output_buffer.buffer();
+                let bps = 4;
+                let u = buf.len() * 2 / 3;
+                let v = buf.len() * 5 / 6;
+                rgb.raw.resize(h * w * bps, 0);
+                let y_ptr = buf.as_ptr();
+                let u_ptr = buf[u..].as_ptr();
+                let v_ptr = buf[v..].as_ptr();
+                unsafe {
+                    match rgb.fmt() {
+                        ImageFormat::ARGB => {
+                            I420ToARGB(
+                                y_ptr,
+                                stride,
+                                u_ptr,
+                                stride / 2,
+                                v_ptr,
+                                stride / 2,
+                                rgb.raw.as_mut_ptr(),
+                                (w * bps) as _,
+                                w as _,
+                                h as _,
+                            );
+                        }
+                        ImageFormat::ARGB => {
+                            I420ToABGR(
+                                y_ptr,
+                                stride,
+                                u_ptr,
+                                stride / 2,
+                                v_ptr,
+                                stride / 2,
+                                rgb.raw.as_mut_ptr(),
+                                (w * bps) as _,
+                                w as _,
+                                h as _,
+                            );
+                        }
+                        _ => {
+                            bail!("Unsupported image format");
+                        }
+                    }
+                }
+                self.release_output_buffer(output_buffer, false)?;
+                Ok(true)
+            }
+            None => {
+                log::debug!("Failed to dequeue_output: No available dequeue_output");
+                Ok(false)
+            }
+        };
+    }
+}
+
+fn create_media_codec(name: &str, direction: MediaCodecDirection) -> Option<MediaCodecDecoder> {
+    let codec = MediaCodec::from_decoder_type(name)?;
+    let media_format = MediaFormat::new();
+    media_format.set_str("mime", name);
+    media_format.set_i32("width", 0);
+    media_format.set_i32("height", 0);
+    media_format.set_i32("color-format", 19); // COLOR_FormatYUV420Planar
+    if let Err(e) = codec.configure(&media_format, None, direction) {
+        log::error!("Failed to init decoder: {:?}", e);
+        return None;
+    };
+    log::error!("decoder init success");
+    if let Err(e) = codec.start() {
+        log::error!("Failed to start decoder: {:?}", e);
+        return None;
+    };
+    log::debug!("Init decoder succeeded!: {:?}", name);
+    return Some(MediaCodecDecoder {
+        decoder: codec,
+        name: name.to_owned(),
+    });
+}
+
+pub fn check_mediacodec() {
+    std::thread::spawn(move || {
+        // check decoders
+        let decoders = MediaCodecDecoder::new_decoders();
+        H264_DECODER_SUPPORT.swap(decoders.h264.is_some(), Ordering::SeqCst);
+        H265_DECODER_SUPPORT.swap(decoders.h265.is_some(), Ordering::SeqCst);
+        decoders.h264.map(|d| d.stop());
+        decoders.h265.map(|d| d.stop());
+        // TODO encoders
+    });
+}

libs/scrap/src/common/mod.rs

@@ -0,0 +1,547 @@
+pub use self::vpxcodec::*;
+use hbb_common::{
+    bail, log,
+    message_proto::{video_frame, Chroma, VideoFrame},
+    ResultType,
+};
+use std::{ffi::c_void, slice};
+
+cfg_if! {
+    if #[cfg(quartz)] {
+        mod quartz;
+        pub use self::quartz::*;
+    } else if #[cfg(x11)] {
+        cfg_if! {
+            if #[cfg(feature="wayland")] {
+                mod linux;
+                mod wayland;
+                mod x11;
+                pub use self::linux::*;
+                pub use self::wayland::set_map_err;
+                pub use self::x11::PixelBuffer;
+            } else {
+                mod x11;
+                pub use self::x11::*;
+            }
+        }
+    } else if #[cfg(dxgi)] {
+        mod dxgi;
+        pub use self::dxgi::*;
+    } else if #[cfg(target_os = "android")] {
+        mod android;
+        pub use self::android::*;
+    }else {
+        //TODO: Fallback implementation.
+    }
+}
+
+pub mod codec;
+pub mod convert;
+#[cfg(feature = "hwcodec")]
+pub mod hwcodec;
+#[cfg(feature = "mediacodec")]
+pub mod mediacodec;
+pub mod vpxcodec;
+#[cfg(feature = "vram")]
+pub mod vram;
+pub use self::convert::*;
+pub const STRIDE_ALIGN: usize = 64; // commonly used in libvpx vpx_img_alloc caller
+pub const HW_STRIDE_ALIGN: usize = 0; // recommended by av_frame_get_buffer
+
+pub mod aom;
+#[cfg(not(any(target_os = "ios")))]
+pub mod camera;
+pub mod record;
+mod vpx;
+
+#[repr(usize)]
+#[derive(Debug, Copy, Clone)]
+pub enum ImageFormat {
+    Raw,
+    ABGR,
+    ARGB,
+}
+
+#[repr(C)]
+#[derive(Clone)]
+pub struct ImageRgb {
+    pub raw: Vec<u8>,
+    pub w: usize,
+    pub h: usize,
+    pub fmt: ImageFormat,
+    pub align: usize,
+}
+
+impl ImageRgb {
+    pub fn new(fmt: ImageFormat, align: usize) -> Self {
+        Self {
+            raw: Vec::new(),
+            w: 0,
+            h: 0,
+            fmt,
+            align,
+        }
+    }
+
+    #[inline]
+    pub fn fmt(&self) -> ImageFormat {
+        self.fmt
+    }
+
+    #[inline]
+    pub fn align(&self) -> usize {
+        self.align
+    }
+
+    #[inline]
+    pub fn set_align(&mut self, align: usize) {
+        self.align = align;
+    }
+}
+
+pub struct ImageTexture {
+    pub texture: *mut c_void,
+    pub w: usize,
+    pub h: usize,
+}
+
+impl Default for ImageTexture {
+    fn default() -> Self {
+        Self {
+            texture: std::ptr::null_mut(),
+            w: 0,
+            h: 0,
+        }
+    }
+}
+
+#[inline]
+pub fn would_block_if_equal(old: &mut Vec<u8>, b: &[u8]) -> std::io::Result<()> {
+    // does this really help?
+    if b == &old[..] {
+        return Err(std::io::ErrorKind::WouldBlock.into());
+    }
+    old.resize(b.len(), 0);
+    old.copy_from_slice(b);
+    Ok(())
+}
+
+pub trait TraitCapturer {
+    // We doesn't support
+    #[cfg(not(any(target_os = "ios")))]
+    fn frame<'a>(&'a mut self, timeout: std::time::Duration) -> std::io::Result<Frame<'a>>;
+
+    #[cfg(windows)]
+    fn is_gdi(&self) -> bool;
+    #[cfg(windows)]
+    fn set_gdi(&mut self) -> bool;
+
+    #[cfg(feature = "vram")]
+    fn device(&self) -> AdapterDevice;
+
+    #[cfg(feature = "vram")]
+    fn set_output_texture(&mut self, texture: bool);
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct AdapterDevice {
+    pub device: *mut c_void,
+    pub vendor_id: ::std::os::raw::c_uint,
+    pub luid: i64,
+}
+
+impl Default for AdapterDevice {
+    fn default() -> Self {
+        Self {
+            device: std::ptr::null_mut(),
+            vendor_id: Default::default(),
+            luid: Default::default(),
+        }
+    }
+}
+
+pub trait TraitPixelBuffer {
+    fn data(&self) -> &[u8];
+
+    fn width(&self) -> usize;
+
+    fn height(&self) -> usize;
+
+    fn stride(&self) -> Vec<usize>;
+
+    fn pixfmt(&self) -> Pixfmt;
+}
+
+#[cfg(not(any(target_os = "ios")))]
+pub enum Frame<'a> {
+    PixelBuffer(PixelBuffer<'a>),
+    Texture((*mut c_void, usize)),
+}
+
+#[cfg(not(any(target_os = "ios")))]
+impl Frame<'_> {
+    pub fn valid<'a>(&'a self) -> bool {
+        match self {
+            Frame::PixelBuffer(pixelbuffer) => !pixelbuffer.data().is_empty(),
+            Frame::Texture((texture, _)) => !texture.is_null(),
+        }
+    }
+
+    pub fn to<'a>(
+        &'a self,
+        yuvfmt: EncodeYuvFormat,
+        yuv: &'a mut Vec<u8>,
+        mid_data: &mut Vec<u8>,
+    ) -> ResultType<EncodeInput<'a>> {
+        match self {
+            Frame::PixelBuffer(pixelbuffer) => {
+                convert_to_yuv(&pixelbuffer, yuvfmt, yuv, mid_data)?;
+                Ok(EncodeInput::YUV(yuv))
+            }
+            Frame::Texture(texture) => Ok(EncodeInput::Texture(*texture)),
+        }
+    }
+}
+
+pub enum EncodeInput<'a> {
+    YUV(&'a [u8]),
+    Texture((*mut c_void, usize)),
+}
+
+impl<'a> EncodeInput<'a> {
+    pub fn yuv(&self) -> ResultType<&'_ [u8]> {
+        match self {
+            Self::YUV(f) => Ok(f),
+            _ => bail!("not pixelfbuffer frame"),
+        }
+    }
+
+    pub fn texture(&self) -> ResultType<(*mut c_void, usize)> {
+        match self {
+            Self::Texture(f) => Ok(*f),
+            _ => bail!("not texture frame"),
+        }
+    }
+}
+
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+pub enum Pixfmt {
+    BGRA,
+    RGBA,
+    RGB565LE,
+    I420,
+    NV12,
+    I444,
+}
+
+impl Pixfmt {
+    pub fn bpp(&self) -> usize {
+        match self {
+            Pixfmt::BGRA | Pixfmt::RGBA => 32,
+            Pixfmt::RGB565LE => 16,
+            Pixfmt::I420 | Pixfmt::NV12 => 12,
+            Pixfmt::I444 => 24,
+        }
+    }
+
+    pub fn bytes_per_pixel(&self) -> usize {
+        (self.bpp() + 7) / 8
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct EncodeYuvFormat {
+    pub pixfmt: Pixfmt,
+    pub w: usize,
+    pub h: usize,
+    pub stride: Vec<usize>,
+    pub u: usize,
+    pub v: usize,
+}
+
+#[cfg(x11)]
+#[inline]
+pub fn is_x11() -> bool {
+    hbb_common::platform::linux::is_x11_or_headless()
+}
+
+#[cfg(x11)]
+#[inline]
+pub fn is_cursor_embedded() -> bool {
+    if is_x11() {
+        x11::IS_CURSOR_EMBEDDED
+    } else {
+        false
+    }
+}
+
+#[cfg(not(x11))]
+#[inline]
+pub fn is_cursor_embedded() -> bool {
+    false
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum CodecName {
+    VP8,
+    VP9,
+    AV1,
+    H264RAM(String),
+    H265RAM(String),
+    H264VRAM,
+    H265VRAM,
+}
+
+#[derive(PartialEq, Debug, Clone, Copy)]
+pub enum CodecFormat {
+    VP8,
+    VP9,
+    AV1,
+    H264,
+    H265,
+    Unknown,
+}
+
+impl From<&VideoFrame> for CodecFormat {
+    fn from(it: &VideoFrame) -> Self {
+        match it.union {
+            Some(video_frame::Union::Vp8s(_)) => CodecFormat::VP8,
+            Some(video_frame::Union::Vp9s(_)) => CodecFormat::VP9,
+            Some(video_frame::Union::Av1s(_)) => CodecFormat::AV1,
+            Some(video_frame::Union::H264s(_)) => CodecFormat::H264,
+            Some(video_frame::Union::H265s(_)) => CodecFormat::H265,
+            _ => CodecFormat::Unknown,
+        }
+    }
+}
+
+impl From<&video_frame::Union> for CodecFormat {
+    fn from(it: &video_frame::Union) -> Self {
+        match it {
+            video_frame::Union::Vp8s(_) => CodecFormat::VP8,
+            video_frame::Union::Vp9s(_) => CodecFormat::VP9,
+            video_frame::Union::Av1s(_) => CodecFormat::AV1,
+            video_frame::Union::H264s(_) => CodecFormat::H264,
+            video_frame::Union::H265s(_) => CodecFormat::H265,
+            _ => CodecFormat::Unknown,
+        }
+    }
+}
+
+impl From<&CodecName> for CodecFormat {
+    fn from(value: &CodecName) -> Self {
+        match value {
+            CodecName::VP8 => Self::VP8,
+            CodecName::VP9 => Self::VP9,
+            CodecName::AV1 => Self::AV1,
+            CodecName::H264RAM(_) | CodecName::H264VRAM => Self::H264,
+            CodecName::H265RAM(_) | CodecName::H265VRAM => Self::H265,
+        }
+    }
+}
+
+impl ToString for CodecFormat {
+    fn to_string(&self) -> String {
+        match self {
+            CodecFormat::VP8 => "VP8".into(),
+            CodecFormat::VP9 => "VP9".into(),
+            CodecFormat::AV1 => "AV1".into(),
+            CodecFormat::H264 => "H264".into(),
+            CodecFormat::H265 => "H265".into(),
+            CodecFormat::Unknown => "Unknown".into(),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub enum Error {
+    FailedCall(String),
+    BadPtr(String),
+}
+
+impl std::fmt::Display for Error {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::result::Result<(), std::fmt::Error> {
+        write!(f, "{:?}", self)
+    }
+}
+
+impl std::error::Error for Error {}
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+#[macro_export]
+macro_rules! generate_call_macro {
+    ($func_name:ident, $allow_err:expr) => {
+        macro_rules! $func_name {
+            ($x:expr) => {{
+                let result = unsafe { $x };
+                let result_int = unsafe { std::mem::transmute::<_, i32>(result) };
+                if result_int != 0 {
+                    let message = format!(
+                        "errcode={} {}:{}:{}:{}",
+                        result_int,
+                        module_path!(),
+                        file!(),
+                        line!(),
+                        column!()
+                    );
+                    if $allow_err {
+                        log::warn!("Failed to call {}, {}", stringify!($func_name), message);
+                    } else {
+                        return Err(crate::Error::FailedCall(message).into());
+                    }
+                }
+                result
+            }};
+        }
+    };
+}
+
+#[macro_export]
+macro_rules! generate_call_ptr_macro {
+    ($func_name:ident) => {
+        macro_rules! $func_name {
+            ($x:expr) => {{
+                let result = unsafe { $x };
+                let result_int = unsafe { std::mem::transmute::<_, isize>(result) };
+                if result_int == 0 {
+                    return Err(crate::Error::BadPtr(format!(
+                        "errcode={} {}:{}:{}:{}",
+                        result_int,
+                        module_path!(),
+                        file!(),
+                        line!(),
+                        column!()
+                    ))
+                    .into());
+                }
+                result
+            }};
+        }
+    };
+}
+
+pub trait GoogleImage {
+    fn width(&self) -> usize;
+    fn height(&self) -> usize;
+    fn stride(&self) -> Vec<i32>;
+    fn planes(&self) -> Vec<*mut u8>;
+    fn chroma(&self) -> Chroma;
+    fn get_bytes_per_row(w: usize, fmt: ImageFormat, align: usize) -> usize {
+        let bytes_per_pixel = match fmt {
+            ImageFormat::Raw => 3,
+            ImageFormat::ARGB | ImageFormat::ABGR => 4,
+        };
+        // https://github.com/lemenkov/libyuv/blob/6900494d90ae095d44405cd4cc3f346971fa69c9/source/convert_argb.cc#L128
+        // https://github.com/lemenkov/libyuv/blob/6900494d90ae095d44405cd4cc3f346971fa69c9/source/convert_argb.cc#L129
+        (w * bytes_per_pixel + align - 1) & !(align - 1)
+    }
+    // rgb [in/out] fmt and stride must be set in ImageRgb
+    fn to(&self, rgb: &mut ImageRgb) {
+        rgb.w = self.width();
+        rgb.h = self.height();
+        let bytes_per_row = Self::get_bytes_per_row(rgb.w, rgb.fmt, rgb.align());
+        rgb.raw.resize(rgb.h * bytes_per_row, 0);
+        let stride = self.stride();
+        let planes = self.planes();
+        unsafe {
+            match (self.chroma(), rgb.fmt()) {
+                (Chroma::I420, ImageFormat::Raw) => {
+                    super::I420ToRAW(
+                        planes[0],
+                        stride[0],
+                        planes[1],
+                        stride[1],
+                        planes[2],
+                        stride[2],
+                        rgb.raw.as_mut_ptr(),
+                        bytes_per_row as _,
+                        self.width() as _,
+                        self.height() as _,
+                    );
+                }
+                (Chroma::I420, ImageFormat::ARGB) => {
+                    super::I420ToARGB(
+                        planes[0],
+                        stride[0],
+                        planes[1],
+                        stride[1],
+                        planes[2],
+                        stride[2],
+                        rgb.raw.as_mut_ptr(),
+                        bytes_per_row as _,
+                        self.width() as _,
+                        self.height() as _,
+                    );
+                }
+                (Chroma::I420, ImageFormat::ABGR) => {
+                    super::I420ToABGR(
+                        planes[0],
+                        stride[0],
+                        planes[1],
+                        stride[1],
+                        planes[2],
+                        stride[2],
+                        rgb.raw.as_mut_ptr(),
+                        bytes_per_row as _,
+                        self.width() as _,
+                        self.height() as _,
+                    );
+                }
+                (Chroma::I444, ImageFormat::ARGB) => {
+                    super::I444ToARGB(
+                        planes[0],
+                        stride[0],
+                        planes[1],
+                        stride[1],
+                        planes[2],
+                        stride[2],
+                        rgb.raw.as_mut_ptr(),
+                        bytes_per_row as _,
+                        self.width() as _,
+                        self.height() as _,
+                    );
+                }
+                (Chroma::I444, ImageFormat::ABGR) => {
+                    super::I444ToABGR(
+                        planes[0],
+                        stride[0],
+                        planes[1],
+                        stride[1],
+                        planes[2],
+                        stride[2],
+                        rgb.raw.as_mut_ptr(),
+                        bytes_per_row as _,
+                        self.width() as _,
+                        self.height() as _,
+                    );
+                }
+                // (Chroma::I444, ImageFormat::Raw), new version libyuv have I444ToRAW
+                _ => log::error!("unsupported pixfmt: {:?}", self.chroma()),
+            }
+        }
+    }
+    fn data(&self) -> (&[u8], &[u8], &[u8]) {
+        unsafe {
+            let stride = self.stride();
+            let planes = self.planes();
+            let h = (self.height() as usize + 1) & !1;
+            let n = stride[0] as usize * h;
+            let y = slice::from_raw_parts(planes[0], n);
+            let n = stride[1] as usize * (h >> 1);
+            let u = slice::from_raw_parts(planes[1], n);
+            let v = slice::from_raw_parts(planes[2], n);
+            (y, u, v)
+        }
+    }
+}
+
+#[cfg(target_os = "android")]
+pub fn screen_size() -> (u16, u16, u16) {
+    SCREEN_SIZE.lock().unwrap().clone()
+}
+
+#[cfg(target_os = "android")]
+pub fn is_start() -> Option<bool> {
+    android::is_start()
+}

libs/scrap/src/common/quartz.rs

@@ -0,0 +1,151 @@
+use crate::{quartz, Frame, Pixfmt};
+use std::marker::PhantomData;
+use std::sync::{Arc, Mutex, TryLockError};
+use std::{io, mem};
+
+pub struct Capturer {
+    inner: quartz::Capturer,
+    frame: Arc<Mutex<Option<quartz::Frame>>>,
+    saved_raw_data: Vec<u8>, // for faster compare and copy
+}
+
+impl Capturer {
+    pub fn new(display: Display) -> io::Result<Capturer> {
+        let frame = Arc::new(Mutex::new(None));
+
+        let f = frame.clone();
+        let inner = quartz::Capturer::new(
+            display.0,
+            display.width(),
+            display.height(),
+            quartz::PixelFormat::Argb8888,
+            Default::default(),
+            move |inner| {
+                if let Ok(mut f) = f.lock() {
+                    *f = Some(inner);
+                }
+            },
+        )
+        .map_err(|_| io::Error::from(io::ErrorKind::Other))?;
+
+        Ok(Capturer {
+            inner,
+            frame,
+            saved_raw_data: Vec::new(),
+        })
+    }
+
+    pub fn width(&self) -> usize {
+        self.inner.width()
+    }
+
+    pub fn height(&self) -> usize {
+        self.inner.height()
+    }
+}
+
+impl crate::TraitCapturer for Capturer {
+    fn frame<'a>(&'a mut self, _timeout_ms: std::time::Duration) -> io::Result<Frame<'a>> {
+        match self.frame.try_lock() {
+            Ok(mut handle) => {
+                let mut frame = None;
+                mem::swap(&mut frame, &mut handle);
+
+                match frame {
+                    Some(mut frame) => {
+                        crate::would_block_if_equal(&mut self.saved_raw_data, frame.inner())?;
+                        frame.surface_to_bgra(self.height());
+                        Ok(Frame::PixelBuffer(PixelBuffer {
+                            frame,
+                            data: PhantomData,
+                            width: self.width(),
+                            height: self.height(),
+                        }))
+                    }
+
+                    None => Err(io::ErrorKind::WouldBlock.into()),
+                }
+            }
+
+            Err(TryLockError::WouldBlock) => Err(io::ErrorKind::WouldBlock.into()),
+
+            Err(TryLockError::Poisoned(..)) => Err(io::ErrorKind::Other.into()),
+        }
+    }
+}
+
+pub struct PixelBuffer<'a> {
+    frame: quartz::Frame,
+    data: PhantomData<&'a [u8]>,
+    width: usize,
+    height: usize,
+}
+
+impl<'a> crate::TraitPixelBuffer for PixelBuffer<'a> {
+    fn data(&self) -> &[u8] {
+        &*self.frame
+    }
+
+    fn width(&self) -> usize {
+        self.width
+    }
+
+    fn height(&self) -> usize {
+        self.height
+    }
+
+    fn stride(&self) -> Vec<usize> {
+        let mut v = Vec::new();
+        v.push(self.frame.stride());
+        v
+    }
+
+    fn pixfmt(&self) -> Pixfmt {
+        Pixfmt::BGRA
+    }
+}
+
+pub struct Display(quartz::Display);
+
+impl Display {
+    pub fn primary() -> io::Result<Display> {
+        Ok(Display(quartz::Display::primary()))
+    }
+
+    pub fn all() -> io::Result<Vec<Display>> {
+        Ok(quartz::Display::online()
+            .map_err(|_| io::Error::from(io::ErrorKind::Other))?
+            .into_iter()
+            .map(Display)
+            .collect())
+    }
+
+    pub fn width(&self) -> usize {
+        self.0.width()
+    }
+
+    pub fn height(&self) -> usize {
+        self.0.height()
+    }
+
+    pub fn scale(&self) -> f64 {
+        self.0.scale()
+    }
+
+    pub fn name(&self) -> String {
+        self.0.id().to_string()
+    }
+
+    pub fn is_online(&self) -> bool {
+        self.0.is_online()
+    }
+
+    pub fn origin(&self) -> (i32, i32) {
+        let o = self.0.bounds().origin;
+        (o.x as _, o.y as _)
+    }
+
+    pub fn is_primary(&self) -> bool {
+        self.0.is_primary()
+    }
+}

libs/scrap/src/common/record.rs

@@ -0,0 +1,423 @@
+use crate::CodecFormat;
+#[cfg(feature = "hwcodec")]
+use hbb_common::anyhow::anyhow;
+use hbb_common::{
+    bail, chrono, log,
+    message_proto::{message, video_frame, EncodedVideoFrame, Message},
+    ResultType,
+};
+#[cfg(feature = "hwcodec")]
+use hwcodec::mux::{MuxContext, Muxer};
+use std::{
+    fs::{File, OpenOptions},
+    io,
+    ops::{Deref, DerefMut},
+    path::PathBuf,
+    sync::mpsc::Sender,
+    time::Instant,
+};
+use webm::mux::{self, Segment, Track, VideoTrack, Writer};
+
+const MIN_SECS: u64 = 1;
+
+#[derive(Debug, Clone)]
+pub struct RecorderContext {
+    pub server: bool,
+    pub id: String,
+    pub dir: String,
+    pub display_idx: usize,
+    pub camera: bool,
+    pub tx: Option<Sender<RecordState>>,
+}
+
+#[derive(Debug, Clone)]
+pub struct RecorderContext2 {
+    pub filename: String,
+    pub width: usize,
+    pub height: usize,
+    pub format: CodecFormat,
+}
+
+impl RecorderContext2 {
+    pub fn set_filename(&mut self, ctx: &RecorderContext) -> ResultType<()> {
+        if !PathBuf::from(&ctx.dir).exists() {
+            std::fs::create_dir_all(&ctx.dir)?;
+        }
+        let file = if ctx.server { "incoming" } else { "outgoing" }.to_string()
+            + "_"
+            + &ctx.id.clone()
+            + &chrono::Local::now().format("_%Y%m%d%H%M%S%3f_").to_string()
+            + &format!(
+                "{}{}_",
+                if ctx.camera { "camera" } else { "display" },
+                ctx.display_idx
+            )
+            + &self.format.to_string().to_lowercase()
+            + if self.format == CodecFormat::VP9
+                || self.format == CodecFormat::VP8
+                || self.format == CodecFormat::AV1
+            {
+                ".webm"
+            } else {
+                ".mp4"
+            };
+        self.filename = PathBuf::from(&ctx.dir)
+            .join(file)
+            .to_string_lossy()
+            .to_string();
+        Ok(())
+    }
+}
+
+unsafe impl Send for Recorder {}
+unsafe impl Sync for Recorder {}
+
+pub trait RecorderApi {
+    fn new(ctx: RecorderContext, ctx2: RecorderContext2) -> ResultType<Self>
+    where
+        Self: Sized;
+    fn write_video(&mut self, frame: &EncodedVideoFrame) -> bool;
+}
+
+#[derive(Debug)]
+pub enum RecordState {
+    NewFile(String),
+    NewFrame,
+    WriteTail,
+    RemoveFile,
+}
+
+pub struct Recorder {
+    pub inner: Option<Box<dyn RecorderApi>>,
+    ctx: RecorderContext,
+    ctx2: Option<RecorderContext2>,
+    pts: Option<i64>,
+    check_failed: bool,
+}
+
+impl Deref for Recorder {
+    type Target = Option<Box<dyn RecorderApi>>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.inner
+    }
+}
+
+impl DerefMut for Recorder {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.inner
+    }
+}
+
+impl Recorder {
+    pub fn new(ctx: RecorderContext) -> ResultType<Self> {
+        Ok(Self {
+            inner: None,
+            ctx,
+            ctx2: None,
+            pts: None,
+            check_failed: false,
+        })
+    }
+
+    fn check(&mut self, w: usize, h: usize, format: CodecFormat) -> ResultType<()> {
+        match self.ctx2 {
+            Some(ref ctx2) => {
+                if ctx2.width != w || ctx2.height != h || ctx2.format != format {
+                    let mut ctx2 = RecorderContext2 {
+                        width: w,
+                        height: h,
+                        format,
+                        filename: Default::default(),
+                    };
+                    ctx2.set_filename(&self.ctx)?;
+                    self.ctx2 = Some(ctx2);
+                    self.inner = None;
+                }
+            }
+            None => {
+                let mut ctx2 = RecorderContext2 {
+                    width: w,
+                    height: h,
+                    format,
+                    filename: Default::default(),
+                };
+                ctx2.set_filename(&self.ctx)?;
+                self.ctx2 = Some(ctx2);
+                self.inner = None;
+            }
+        }
+        let Some(ctx2) = &self.ctx2 else {
+            bail!("ctx2 is None");
+        };
+        if self.inner.is_none() {
+            self.inner = match format {
+                CodecFormat::VP8 | CodecFormat::VP9 | CodecFormat::AV1 => Some(Box::new(
+                    WebmRecorder::new(self.ctx.clone(), (*ctx2).clone())?,
+                )),
+                #[cfg(feature = "hwcodec")]
+                _ => Some(Box::new(HwRecorder::new(
+                    self.ctx.clone(),
+                    (*ctx2).clone(),
+                )?)),
+                #[cfg(not(feature = "hwcodec"))]
+                _ => bail!("unsupported codec type"),
+            };
+            // pts is None when new inner is created
+            self.pts = None;
+            self.send_state(RecordState::NewFile(ctx2.filename.clone()));
+        }
+        Ok(())
+    }
+
+    pub fn write_message(&mut self, msg: &Message, w: usize, h: usize) {
+        if let Some(message::Union::VideoFrame(vf)) = &msg.union {
+            if let Some(frame) = &vf.union {
+                self.write_frame(frame, w, h).ok();
+            }
+        }
+    }
+
+    pub fn write_frame(
+        &mut self,
+        frame: &video_frame::Union,
+        w: usize,
+        h: usize,
+    ) -> ResultType<()> {
+        if self.check_failed {
+            bail!("check failed");
+        }
+        let format = CodecFormat::from(frame);
+        if format == CodecFormat::Unknown {
+            bail!("unsupported frame type");
+        }
+        let res = self.check(w, h, format);
+        if res.is_err() {
+            self.check_failed = true;
+            log::error!("check failed: {:?}", res);
+            res?;
+        }
+        match frame {
+            video_frame::Union::Vp8s(vp8s) => {
+                for f in vp8s.frames.iter() {
+                    self.check_pts(f.pts, f.key, w, h, format)?;
+                    self.as_mut().map(|x| x.write_video(f));
+                }
+            }
+            video_frame::Union::Vp9s(vp9s) => {
+                for f in vp9s.frames.iter() {
+                    self.check_pts(f.pts, f.key, w, h, format)?;
+                    self.as_mut().map(|x| x.write_video(f));
+                }
+            }
+            video_frame::Union::Av1s(av1s) => {
+                for f in av1s.frames.iter() {
+                    self.check_pts(f.pts, f.key, w, h, format)?;
+                    self.as_mut().map(|x| x.write_video(f));
+                }
+            }
+            #[cfg(feature = "hwcodec")]
+            video_frame::Union::H264s(h264s) => {
+                for f in h264s.frames.iter() {
+                    self.check_pts(f.pts, f.key, w, h, format)?;
+                    self.as_mut().map(|x| x.write_video(f));
+                }
+            }
+            #[cfg(feature = "hwcodec")]
+            video_frame::Union::H265s(h265s) => {
+                for f in h265s.frames.iter() {
+                    self.check_pts(f.pts, f.key, w, h, format)?;
+                    self.as_mut().map(|x| x.write_video(f));
+                }
+            }
+            _ => bail!("unsupported frame type"),
+        }
+        self.send_state(RecordState::NewFrame);
+        Ok(())
+    }
+
+    fn check_pts(
+        &mut self,
+        pts: i64,
+        key: bool,
+        w: usize,
+        h: usize,
+        format: CodecFormat,
+    ) -> ResultType<()> {
+        // https://stackoverflow.com/questions/76379101/how-to-create-one-playable-webm-file-from-two-different-video-tracks-with-same-c
+        if self.pts.is_none() && !key {
+            bail!("first frame is not key frame");
+        }
+        let old_pts = self.pts;
+        self.pts = Some(pts);
+        if old_pts.clone().unwrap_or_default() > pts {
+            log::info!("pts {:?} -> {}, change record filename", old_pts, pts);
+            self.inner = None;
+            self.ctx2 = None;
+            let res = self.check(w, h, format);
+            if res.is_err() {
+                self.check_failed = true;
+                log::error!("check failed: {:?}", res);
+                res?;
+            }
+            self.pts = Some(pts);
+        }
+        Ok(())
+    }
+
+    fn send_state(&self, state: RecordState) {
+        self.ctx.tx.as_ref().map(|tx| tx.send(state));
+    }
+}
+
+struct WebmRecorder {
+    vt: VideoTrack,
+    webm: Option<Segment<Writer<File>>>,
+    ctx: RecorderContext,
+    ctx2: RecorderContext2,
+    key: bool,
+    written: bool,
+    start: Instant,
+}
+
+impl RecorderApi for WebmRecorder {
+    fn new(ctx: RecorderContext, ctx2: RecorderContext2) -> ResultType<Self> {
+        let out = match {
+            OpenOptions::new()
+                .write(true)
+                .create_new(true)
+                .open(&ctx2.filename)
+        } {
+            Ok(file) => file,
+            Err(ref e) if e.kind() == io::ErrorKind::AlreadyExists => File::create(&ctx2.filename)?,
+            Err(e) => return Err(e.into()),
+        };
+        let mut webm = match mux::Segment::new(mux::Writer::new(out)) {
+            Some(v) => v,
+            None => bail!("Failed to create webm mux"),
+        };
+        let vt = webm.add_video_track(
+            ctx2.width as _,
+            ctx2.height as _,
+            None,
+            if ctx2.format == CodecFormat::VP9 {
+                mux::VideoCodecId::VP9
+            } else if ctx2.format == CodecFormat::VP8 {
+                mux::VideoCodecId::VP8
+            } else {
+                mux::VideoCodecId::AV1
+            },
+        );
+        if ctx2.format == CodecFormat::AV1 {
+            // [129, 8, 12, 0] in 3.6.0, but zero works
+            let codec_private = vec![0, 0, 0, 0];
+            if !webm.set_codec_private(vt.track_number(), &codec_private) {
+                bail!("Failed to set codec private");
+            }
+        }
+        Ok(WebmRecorder {
+            vt,
+            webm: Some(webm),
+            ctx,
+            ctx2,
+            key: false,
+            written: false,
+            start: Instant::now(),
+        })
+    }
+
+    fn write_video(&mut self, frame: &EncodedVideoFrame) -> bool {
+        if frame.key {
+            self.key = true;
+        }
+        if self.key {
+            let ok = self
+                .vt
+                .add_frame(&frame.data, frame.pts as u64 * 1_000_000, frame.key);
+            if ok {
+                self.written = true;
+            }
+            ok
+        } else {
+            false
+        }
+    }
+}
+
+impl Drop for WebmRecorder {
+    fn drop(&mut self) {
+        let _ = std::mem::replace(&mut self.webm, None).map_or(false, |webm| webm.finalize(None));
+        let mut state = RecordState::WriteTail;
+        if !self.written || self.start.elapsed().as_secs() < MIN_SECS {
+            std::fs::remove_file(&self.ctx2.filename).ok();
+            state = RecordState::RemoveFile;
+        }
+        self.ctx.tx.as_ref().map(|tx| tx.send(state));
+    }
+}
+
+#[cfg(feature = "hwcodec")]
+struct HwRecorder {
+    muxer: Option<Muxer>,
+    ctx: RecorderContext,
+    ctx2: RecorderContext2,
+    written: bool,
+    key: bool,
+    start: Instant,
+}
+
+#[cfg(feature = "hwcodec")]
+impl RecorderApi for HwRecorder {
+    fn new(ctx: RecorderContext, ctx2: RecorderContext2) -> ResultType<Self> {
+        let muxer = Muxer::new(MuxContext {
+            filename: ctx2.filename.clone(),
+            width: ctx2.width,
+            height: ctx2.height,
+            is265: ctx2.format == CodecFormat::H265,
+            framerate: crate::hwcodec::DEFAULT_FPS as _,
+        })
+        .map_err(|_| anyhow!("Failed to create hardware muxer"))?;
+        Ok(HwRecorder {
+            muxer: Some(muxer),
+            ctx,
+            ctx2,
+            written: false,
+            key: false,
+            start: Instant::now(),
+        })
+    }
+
+    fn write_video(&mut self, frame: &EncodedVideoFrame) -> bool {
+        if frame.key {
+            self.key = true;
+        }
+        if self.key {
+            let ok = self
+                .muxer
+                .as_mut()
+                .map(|m| m.write_video(&frame.data, frame.key).is_ok())
+                .unwrap_or_default();
+            if ok {
+                self.written = true;
+            }
+            ok
+        } else {
+            false
+        }
+    }
+}
+
+#[cfg(feature = "hwcodec")]
+impl Drop for HwRecorder {
+    fn drop(&mut self) {
+        self.muxer.as_mut().map(|m| m.write_tail().ok());
+        let mut state = RecordState::WriteTail;
+        if !self.written || self.start.elapsed().as_secs() < MIN_SECS {
+            // The process cannot access the file because it is being used by another process
+            self.muxer = None;
+            std::fs::remove_file(&self.ctx2.filename).ok();
+            state = RecordState::RemoveFile;
+        }
+        self.ctx.tx.as_ref().map(|tx| tx.send(state));
+    }
+}

libs/scrap/src/common/vpx.rs

@@ -0,0 +1,26 @@
+#![allow(non_camel_case_types)]
+#![allow(non_snake_case)]
+#![allow(non_upper_case_globals)]
+#![allow(improper_ctypes)]
+#![allow(dead_code)]
+#![allow(unused_imports)]
+
+impl Default for vpx_codec_enc_cfg {
+    fn default() -> Self {
+        unsafe { std::mem::zeroed() }
+    }
+}
+
+impl Default for vpx_codec_ctx {
+    fn default() -> Self {
+        unsafe { std::mem::zeroed() }
+    }
+}
+
+impl Default for vpx_image_t {
+    fn default() -> Self {
+        unsafe { std::mem::zeroed() }
+    }
+}
+
+include!(concat!(env!("OUT_DIR"), "/vpx_ffi.rs"));

libs/scrap/src/common/vpxcodec.rs

@@ -0,0 +1,597 @@
+// https://github.com/astraw/vpx-encode
+// https://github.com/astraw/env-libvpx-sys
+// https://github.com/rust-av/vpx-rs/blob/master/src/decoder.rs
+// https://github.com/chromium/chromium/blob/e7b24573bc2e06fed4749dd6b6abfce67f29052f/media/video/vpx_video_encoder.cc#L522
+
+use hbb_common::anyhow::{anyhow, Context};
+use hbb_common::log;
+use hbb_common::message_proto::{Chroma, EncodedVideoFrame, EncodedVideoFrames, VideoFrame};
+use hbb_common::ResultType;
+
+use crate::codec::{base_bitrate, codec_thread_num, EncoderApi};
+use crate::{EncodeInput, EncodeYuvFormat, GoogleImage, Pixfmt, STRIDE_ALIGN};
+
+use super::vpx::{vp8e_enc_control_id::*, vpx_codec_err_t::*, *};
+use crate::{generate_call_macro, generate_call_ptr_macro, Error, Result};
+use hbb_common::bytes::Bytes;
+use std::os::raw::{c_int, c_uint};
+use std::{ptr, slice};
+
+generate_call_macro!(call_vpx, false);
+generate_call_ptr_macro!(call_vpx_ptr);
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
+pub enum VpxVideoCodecId {
+    VP8,
+    VP9,
+}
+
+impl Default for VpxVideoCodecId {
+    fn default() -> VpxVideoCodecId {
+        VpxVideoCodecId::VP9
+    }
+}
+
+pub struct VpxEncoder {
+    ctx: vpx_codec_ctx_t,
+    width: usize,
+    height: usize,
+    id: VpxVideoCodecId,
+    i444: bool,
+    yuvfmt: EncodeYuvFormat,
+}
+
+pub struct VpxDecoder {
+    ctx: vpx_codec_ctx_t,
+}
+
+impl EncoderApi for VpxEncoder {
+    fn new(cfg: crate::codec::EncoderCfg, i444: bool) -> ResultType<Self>
+    where
+        Self: Sized,
+    {
+        match cfg {
+            crate::codec::EncoderCfg::VPX(config) => {
+                let i = match config.codec {
+                    VpxVideoCodecId::VP8 => call_vpx_ptr!(vpx_codec_vp8_cx()),
+                    VpxVideoCodecId::VP9 => call_vpx_ptr!(vpx_codec_vp9_cx()),
+                };
+                let mut c = unsafe { std::mem::MaybeUninit::zeroed().assume_init() };
+                call_vpx!(vpx_codec_enc_config_default(i, &mut c, 0));
+
+                // https://www.webmproject.org/docs/encoder-parameters/
+                // default: c.rc_min_quantizer = 0, c.rc_max_quantizer = 63
+                // try rc_resize_allowed later
+
+                c.g_w = config.width;
+                c.g_h = config.height;
+                c.g_timebase.num = 1;
+                c.g_timebase.den = 1000; // Output timestamp precision
+                c.rc_undershoot_pct = 95;
+                // When the data buffer falls below this percentage of fullness, a dropped frame is indicated. Set the threshold to zero (0) to disable this feature.
+                // In dynamic scenes, low bitrate gets low fps while high bitrate gets high fps.
+                c.rc_dropframe_thresh = 25;
+                c.g_threads = codec_thread_num(64) as _;
+                c.g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT;
+                // https://developers.google.com/media/vp9/bitrate-modes/
+                // Constant Bitrate mode (CBR) is recommended for live streaming with VP9.
+                c.rc_end_usage = vpx_rc_mode::VPX_CBR;
+                if let Some(keyframe_interval) = config.keyframe_interval {
+                    c.kf_min_dist = 0;
+                    c.kf_max_dist = keyframe_interval as _;
+                } else {
+                    c.kf_mode = vpx_kf_mode::VPX_KF_DISABLED; // reduce bandwidth a lot
+                }
+
+                let (q_min, q_max) = Self::calc_q_values(config.quality);
+                c.rc_min_quantizer = q_min;
+                c.rc_max_quantizer = q_max;
+                c.rc_target_bitrate =
+                    Self::bitrate(config.width as _, config.height as _, config.quality);
+                // https://chromium.googlesource.com/webm/libvpx/+/refs/heads/main/vp9/common/vp9_enums.h#29
+                // https://chromium.googlesource.com/webm/libvpx/+/refs/heads/main/vp8/vp8_cx_iface.c#282
+                c.g_profile = if i444 && config.codec == VpxVideoCodecId::VP9 {
+                    1
+                } else {
+                    0
+                };
+
+                /*
+                The VPX encoder supports two-pass encoding for rate control purposes.
+                In two-pass encoding, the entire encoding process is performed twice.
+                The first pass generates new control parameters for the second pass.
+
+                This approach enables the best PSNR at the same bit rate.
+                */
+
+                let mut ctx = Default::default();
+                call_vpx!(vpx_codec_enc_init_ver(
+                    &mut ctx,
+                    i,
+                    &c,
+                    0,
+                    VPX_ENCODER_ABI_VERSION as _
+                ));
+
+                if config.codec == VpxVideoCodecId::VP9 {
+                    // set encoder internal speed settings
+                    // in ffmpeg, it is --speed option
+                    /*
+                    set to 0 or a positive value 1-16, the codec will try to adapt its
+                    complexity depending on the time it spends encoding. Increasing this
+                    number will make the speed go up and the quality go down.
+                    Negative values mean strict enforcement of this
+                    while positive values are adaptive
+                    */
+                    /* https://developers.google.com/media/vp9/live-encoding
+                    Speed 5 to 8 should be used for live / real-time encoding.
+                    Lower numbers (5 or 6) are higher quality but require more CPU power.
+                    Higher numbers (7 or 8) will be lower quality but more manageable for lower latency
+                    use cases and also for lower CPU power devices such as mobile.
+                    */
+                    call_vpx!(vpx_codec_control_(&mut ctx, VP8E_SET_CPUUSED as _, 7,));
+                    // set row level multi-threading
+                    /*
+                    as some people in comments and below have already commented,
+                    more recent versions of libvpx support -row-mt 1 to enable tile row
+                    multi-threading. This can increase the number of tiles by up to 4x in VP9
+                    (since the max number of tile rows is 4, regardless of video height).
+                    To enable this, use -tile-rows N where N is the number of tile rows in
+                    log2 units (so -tile-rows 1 means 2 tile rows and -tile-rows 2 means 4 tile
+                    rows). The total number of active threads will then be equal to
+                    $tile_rows * $tile_columns
+                    */
+                    call_vpx!(vpx_codec_control_(
+                        &mut ctx,
+                        VP9E_SET_ROW_MT as _,
+                        1 as c_int
+                    ));
+
+                    call_vpx!(vpx_codec_control_(
+                        &mut ctx,
+                        VP9E_SET_TILE_COLUMNS as _,
+                        4 as c_int
+                    ));
+                } else if config.codec == VpxVideoCodecId::VP8 {
+                    // https://github.com/webmproject/libvpx/blob/972149cafeb71d6f08df89e91a0130d6a38c4b15/vpx/vp8cx.h#L172
+                    // https://groups.google.com/a/webmproject.org/g/webm-discuss/c/DJhSrmfQ61M
+                    call_vpx!(vpx_codec_control_(&mut ctx, VP8E_SET_CPUUSED as _, 12,));
+                }
+
+                Ok(Self {
+                    ctx,
+                    width: config.width as _,
+                    height: config.height as _,
+                    id: config.codec,
+                    i444,
+                    yuvfmt: Self::get_yuvfmt(config.width, config.height, i444),
+                })
+            }
+            _ => Err(anyhow!("encoder type mismatch")),
+        }
+    }
+
+    fn encode_to_message(&mut self, input: EncodeInput, ms: i64) -> ResultType<VideoFrame> {
+        let mut frames = Vec::new();
+        for ref frame in self
+            .encode(ms, input.yuv()?, STRIDE_ALIGN)
+            .with_context(|| "Failed to encode")?
+        {
+            frames.push(VpxEncoder::create_frame(frame));
+        }
+        for ref frame in self.flush().with_context(|| "Failed to flush")? {
+            frames.push(VpxEncoder::create_frame(frame));
+        }
+
+        // to-do: flush periodically, e.g. 1 second
+        if frames.len() > 0 {
+            Ok(VpxEncoder::create_video_frame(self.id, frames))
+        } else {
+            Err(anyhow!("no valid frame"))
+        }
+    }
+
+    fn yuvfmt(&self) -> crate::EncodeYuvFormat {
+        self.yuvfmt.clone()
+    }
+
+    #[cfg(feature = "vram")]
+    fn input_texture(&self) -> bool {
+        false
+    }
+
+    fn set_quality(&mut self, ratio: f32) -> ResultType<()> {
+        let mut c = unsafe { *self.ctx.config.enc.to_owned() };
+        let (q_min, q_max) = Self::calc_q_values(ratio);
+        c.rc_min_quantizer = q_min;
+        c.rc_max_quantizer = q_max;
+        c.rc_target_bitrate = Self::bitrate(self.width as _, self.height as _, ratio);
+        call_vpx!(vpx_codec_enc_config_set(&mut self.ctx, &c));
+        Ok(())
+    }
+
+    fn bitrate(&self) -> u32 {
+        let c = unsafe { *self.ctx.config.enc.to_owned() };
+        c.rc_target_bitrate
+    }
+
+    fn support_changing_quality(&self) -> bool {
+        true
+    }
+
+    fn latency_free(&self) -> bool {
+        true
+    }
+
+    fn is_hardware(&self) -> bool {
+        false
+    }
+
+    fn disable(&self) {}
+}
+
+impl VpxEncoder {
+    pub fn encode<'a>(&'a mut self, pts: i64, data: &[u8], stride_align: usize) -> Result<EncodeFrames<'a>> {
+        let bpp = if self.i444 { 24 } else { 12 };
+        if data.len() < self.width * self.height * bpp / 8 {
+            return Err(Error::FailedCall("len not enough".to_string()));
+        }
+        let fmt = if self.i444 {
+            vpx_img_fmt::VPX_IMG_FMT_I444
+        } else {
+            vpx_img_fmt::VPX_IMG_FMT_I420
+        };
+
+        let mut image = Default::default();
+        call_vpx_ptr!(vpx_img_wrap(
+            &mut image,
+            fmt,
+            self.width as _,
+            self.height as _,
+            stride_align as _,
+            data.as_ptr() as _,
+        ));
+
+        call_vpx!(vpx_codec_encode(
+            &mut self.ctx,
+            &image,
+            pts as _,
+            1, // Duration
+            0, // Flags
+            VPX_DL_REALTIME as _,
+        ));
+
+        Ok(EncodeFrames {
+            ctx: &mut self.ctx,
+            iter: ptr::null(),
+        })
+    }
+
+    /// Notify the encoder to return any pending packets
+    pub fn flush<'a>(&'a mut self) -> Result<EncodeFrames<'a>> {
+        call_vpx!(vpx_codec_encode(
+            &mut self.ctx,
+            ptr::null(),
+            -1, // PTS
+            1,  // Duration
+            0,  // Flags
+            VPX_DL_REALTIME as _,
+        ));
+
+        Ok(EncodeFrames {
+            ctx: &mut self.ctx,
+            iter: ptr::null(),
+        })
+    }
+
+    #[inline]
+    pub fn create_video_frame(
+        codec_id: VpxVideoCodecId,
+        frames: Vec<EncodedVideoFrame>,
+    ) -> VideoFrame {
+        let mut vf = VideoFrame::new();
+        let vpxs = EncodedVideoFrames {
+            frames: frames.into(),
+            ..Default::default()
+        };
+        match codec_id {
+            VpxVideoCodecId::VP8 => vf.set_vp8s(vpxs),
+            VpxVideoCodecId::VP9 => vf.set_vp9s(vpxs),
+        }
+        vf
+    }
+
+    #[inline]
+    fn create_frame(frame: &EncodeFrame) -> EncodedVideoFrame {
+        EncodedVideoFrame {
+            data: Bytes::from(frame.data.to_vec()),
+            key: frame.key,
+            pts: frame.pts,
+            ..Default::default()
+        }
+    }
+
+    fn bitrate(width: u32, height: u32, ratio: f32) -> u32 {
+        let bitrate = base_bitrate(width, height) as f32;
+        (bitrate * ratio) as u32
+    }
+
+    #[inline]
+    fn calc_q_values(ratio: f32) -> (u32, u32) {
+        let b = (ratio * 100.0) as u32;
+        let b = std::cmp::min(b, 200);
+        let q_min1 = 36;
+        let q_min2 = 0;
+        let q_max1 = 56;
+        let q_max2 = 37;
+
+        let t = b as f32 / 200.0;
+
+        let mut q_min: u32 = ((1.0 - t) * q_min1 as f32 + t * q_min2 as f32).round() as u32;
+        let mut q_max = ((1.0 - t) * q_max1 as f32 + t * q_max2 as f32).round() as u32;
+
+        q_min = q_min.clamp(q_min2, q_min1);
+        q_max = q_max.clamp(q_max2, q_max1);
+
+        (q_min, q_max)
+    }
+
+    fn get_yuvfmt(width: u32, height: u32, i444: bool) -> EncodeYuvFormat {
+        let mut img = Default::default();
+        let fmt = if i444 {
+            vpx_img_fmt::VPX_IMG_FMT_I444
+        } else {
+            vpx_img_fmt::VPX_IMG_FMT_I420
+        };
+        unsafe {
+            vpx_img_wrap(
+                &mut img,
+                fmt,
+                width as _,
+                height as _,
+                crate::STRIDE_ALIGN as _,
+                0x1 as _,
+            );
+        }
+        let pixfmt = if i444 { Pixfmt::I444 } else { Pixfmt::I420 };
+        EncodeYuvFormat {
+            pixfmt,
+            w: img.w as _,
+            h: img.h as _,
+            stride: img.stride.map(|s| s as usize).to_vec(),
+            u: img.planes[1] as usize - img.planes[0] as usize,
+            v: img.planes[2] as usize - img.planes[0] as usize,
+        }
+    }
+}
+
+impl Drop for VpxEncoder {
+    fn drop(&mut self) {
+        unsafe {
+            let result = vpx_codec_destroy(&mut self.ctx);
+            if result != VPX_CODEC_OK {
+                panic!("failed to destroy vpx codec");
+            }
+        }
+    }
+}
+
+#[derive(Clone, Copy, Debug)]
+pub struct EncodeFrame<'a> {
+    /// Compressed data.
+    pub data: &'a [u8],
+    /// Whether the frame is a keyframe.
+    pub key: bool,
+    /// Presentation timestamp (in timebase units).
+    pub pts: i64,
+}
+
+#[derive(Clone, Copy, Debug)]
+pub struct VpxEncoderConfig {
+    /// The width (in pixels).
+    pub width: c_uint,
+    /// The height (in pixels).
+    pub height: c_uint,
+    /// The bitrate ratio
+    pub quality: f32,
+    /// The codec
+    pub codec: VpxVideoCodecId,
+    /// keyframe interval
+    pub keyframe_interval: Option<usize>,
+}
+
+#[derive(Clone, Copy, Debug)]
+pub struct VpxDecoderConfig {
+    pub codec: VpxVideoCodecId,
+}
+
+pub struct EncodeFrames<'a> {
+    ctx: &'a mut vpx_codec_ctx_t,
+    iter: vpx_codec_iter_t,
+}
+
+impl<'a> Iterator for EncodeFrames<'a> {
+    type Item = EncodeFrame<'a>;
+    fn next(&mut self) -> Option<Self::Item> {
+        loop {
+            unsafe {
+                let pkt = vpx_codec_get_cx_data(self.ctx, &mut self.iter);
+                if pkt.is_null() {
+                    return None;
+                } else if (*pkt).kind == vpx_codec_cx_pkt_kind::VPX_CODEC_CX_FRAME_PKT {
+                    let f = &(*pkt).data.frame;
+                    return Some(Self::Item {
+                        data: slice::from_raw_parts(f.buf as _, f.sz as _),
+                        key: (f.flags & VPX_FRAME_IS_KEY) != 0,
+                        pts: f.pts,
+                    });
+                } else {
+                    // Ignore the packet.
+                }
+            }
+        }
+    }
+}
+
+impl VpxDecoder {
+    /// Create a new decoder
+    ///
+    /// # Errors
+    ///
+    /// The function may fail if the underlying libvpx does not provide
+    /// the VP9 decoder.
+    pub fn new(config: VpxDecoderConfig) -> Result<Self> {
+        // This is sound because `vpx_codec_ctx` is a repr(C) struct without any field that can
+        // cause UB if uninitialized.
+        let i = match config.codec {
+            VpxVideoCodecId::VP8 => call_vpx_ptr!(vpx_codec_vp8_dx()),
+            VpxVideoCodecId::VP9 => call_vpx_ptr!(vpx_codec_vp9_dx()),
+        };
+        let mut ctx = Default::default();
+        let cfg = vpx_codec_dec_cfg_t {
+            threads: codec_thread_num(64) as _,
+            w: 0,
+            h: 0,
+        };
+        /*
+        unsafe {
+            println!("{}", vpx_codec_get_caps(i));
+        }
+        */
+        call_vpx!(vpx_codec_dec_init_ver(
+            &mut ctx,
+            i,
+            &cfg,
+            0,
+            VPX_DECODER_ABI_VERSION as _,
+        ));
+        Ok(Self { ctx })
+    }
+
+    /// Feed some compressed data to the encoder
+    ///
+    /// The `data` slice is sent to the decoder
+    ///
+    /// It matches a call to `vpx_codec_decode`.
+    pub fn decode<'a>(&'a mut self, data: &[u8]) -> Result<DecodeFrames<'a>> {
+        call_vpx!(vpx_codec_decode(
+            &mut self.ctx,
+            data.as_ptr(),
+            data.len() as _,
+            ptr::null_mut(),
+            0,
+        ));
+
+        Ok(DecodeFrames {
+            ctx: &mut self.ctx,
+            iter: ptr::null(),
+        })
+    }
+
+    /// Notify the decoder to return any pending frame
+    pub fn flush<'a>(&'a mut self) -> Result<DecodeFrames<'a>> {
+        call_vpx!(vpx_codec_decode(
+            &mut self.ctx,
+            ptr::null(),
+            0,
+            ptr::null_mut(),
+            0
+        ));
+        Ok(DecodeFrames {
+            ctx: &mut self.ctx,
+            iter: ptr::null(),
+        })
+    }
+}
+
+impl Drop for VpxDecoder {
+    fn drop(&mut self) {
+        unsafe {
+            let result = vpx_codec_destroy(&mut self.ctx);
+            if result != VPX_CODEC_OK {
+                panic!("failed to destroy vpx codec");
+            }
+        }
+    }
+}
+
+pub struct DecodeFrames<'a> {
+    ctx: &'a mut vpx_codec_ctx_t,
+    iter: vpx_codec_iter_t,
+}
+
+impl<'a> Iterator for DecodeFrames<'a> {
+    type Item = Image;
+    fn next(&mut self) -> Option<Self::Item> {
+        let img = unsafe { vpx_codec_get_frame(self.ctx, &mut self.iter) };
+        if img.is_null() {
+            return None;
+        } else {
+            return Some(Image(img));
+        }
+    }
+}
+
+// https://chromium.googlesource.com/webm/libvpx/+/bali/vpx/src/vpx_image.c
+pub struct Image(*mut vpx_image_t);
+impl Image {
+    #[inline]
+    pub fn new() -> Self {
+        Self(std::ptr::null_mut())
+    }
+
+    #[inline]
+    pub fn is_null(&self) -> bool {
+        self.0.is_null()
+    }
+
+    #[inline]
+    pub fn format(&self) -> vpx_img_fmt_t {
+        // VPX_IMG_FMT_I420
+        self.inner().fmt
+    }
+
+    #[inline]
+    pub fn inner(&self) -> &vpx_image_t {
+        unsafe { &*self.0 }
+    }
+}
+
+impl GoogleImage for Image {
+    #[inline]
+    fn width(&self) -> usize {
+        self.inner().d_w as _
+    }
+
+    #[inline]
+    fn height(&self) -> usize {
+        self.inner().d_h as _
+    }
+
+    #[inline]
+    fn stride(&self) -> Vec<i32> {
+        self.inner().stride.iter().map(|x| *x as i32).collect()
+    }
+
+    #[inline]
+    fn planes(&self) -> Vec<*mut u8> {
+        self.inner().planes.iter().map(|p| *p as *mut u8).collect()
+    }
+
+    fn chroma(&self) -> Chroma {
+        match self.inner().fmt {
+            vpx_img_fmt::VPX_IMG_FMT_I444 => Chroma::I444,
+            _ => Chroma::I420,
+        }
+    }
+}
+
+impl Drop for Image {
+    fn drop(&mut self) {
+        if !self.0.is_null() {
+            unsafe { vpx_img_free(self.0) };
+        }
+    }
+}
+
+unsafe impl Send for vpx_codec_ctx_t {}

libs/scrap/src/common/vram.rs

@@ -0,0 +1,404 @@
+use std::{
+    collections::{HashMap, HashSet},
+    ffi::c_void,
+    sync::{Arc, Mutex},
+};
+
+use crate::{
+    codec::{enable_vram_option, EncoderApi, EncoderCfg},
+    hwcodec::HwCodecConfig,
+    AdapterDevice, CodecFormat, EncodeInput, EncodeYuvFormat, Pixfmt,
+};
+use hbb_common::{
+    anyhow::{anyhow, bail, Context},
+    bytes::Bytes,
+    log,
+    message_proto::{EncodedVideoFrame, EncodedVideoFrames, VideoFrame},
+    ResultType,
+};
+use hwcodec::{
+    common::{DataFormat, Driver, MAX_GOP},
+    vram::{
+        decode::{self, DecodeFrame, Decoder},
+        encode::{self, EncodeFrame, Encoder},
+        Available, DecodeContext, DynamicContext, EncodeContext, FeatureContext,
+    },
+};
+
+// https://www.reddit.com/r/buildapc/comments/d2m4ny/two_graphics_cards_two_monitors/
+// https://www.reddit.com/r/techsupport/comments/t2v9u6/dual_monitor_setup_with_dual_gpu/
+// https://cybersided.com/two-monitors-two-gpus/
+// https://learn.microsoft.com/en-us/windows/win32/api/d3d12/nf-d3d12-id3d12device-getadapterluid#remarks
+lazy_static::lazy_static! {
+    static ref ENOCDE_NOT_USE: Arc<Mutex<HashMap<String, bool>>> = Default::default();
+    static ref FALLBACK_GDI_DISPLAYS: Arc<Mutex<HashSet<String>>> = Default::default();
+}
+
+#[derive(Debug, Clone)]
+pub struct VRamEncoderConfig {
+    pub device: AdapterDevice,
+    pub width: usize,
+    pub height: usize,
+    pub quality: f32,
+    pub feature: FeatureContext,
+    pub keyframe_interval: Option<usize>,
+}
+
+pub struct VRamEncoder {
+    encoder: Encoder,
+    pub format: DataFormat,
+    ctx: EncodeContext,
+    bitrate: u32,
+    last_frame_len: usize,
+    same_bad_len_counter: usize,
+}
+
+impl EncoderApi for VRamEncoder {
+    fn new(cfg: EncoderCfg, _i444: bool) -> ResultType<Self>
+    where
+        Self: Sized,
+    {
+        match cfg {
+            EncoderCfg::VRAM(config) => {
+                let bitrate = Self::bitrate(
+                    config.feature.data_format,
+                    config.width,
+                    config.height,
+                    config.quality,
+                );
+                let gop = config.keyframe_interval.unwrap_or(MAX_GOP as _) as i32;
+                let ctx = EncodeContext {
+                    f: config.feature.clone(),
+                    d: DynamicContext {
+                        device: Some(config.device.device),
+                        width: config.width as _,
+                        height: config.height as _,
+                        kbitrate: bitrate as _,
+                        framerate: 30,
+                        gop,
+                    },
+                };
+                match Encoder::new(ctx.clone()) {
+                    Ok(encoder) => Ok(VRamEncoder {
+                        encoder,
+                        ctx,
+                        format: config.feature.data_format,
+                        bitrate,
+                        last_frame_len: 0,
+                        same_bad_len_counter: 0,
+                    }),
+                    Err(_) => Err(anyhow!(format!("Failed to create encoder"))),
+                }
+            }
+            _ => Err(anyhow!("encoder type mismatch")),
+        }
+    }
+
+    fn encode_to_message(
+        &mut self,
+        frame: EncodeInput,
+        ms: i64,
+    ) -> ResultType<hbb_common::message_proto::VideoFrame> {
+        let (texture, rotation) = frame.texture()?;
+        if rotation != 0 {
+            // to-do: support rotation
+            // Both the encoder and display(w,h) information need to be changed.
+            bail!("rotation not supported");
+        }
+        let mut vf = VideoFrame::new();
+        let mut frames = Vec::new();
+        for frame in self
+            .encode(texture, ms)
+            .with_context(|| "Failed to encode")?
+        {
+            frames.push(EncodedVideoFrame {
+                data: Bytes::from(frame.data),
+                pts: frame.pts,
+                key: frame.key == 1,
+                ..Default::default()
+            });
+        }
+        if frames.len() > 0 {
+            // This kind of problem is occurred after a period of time when using AMD encoding,
+            // the encoding length is fixed at about 40, and the picture is still
+            const MIN_BAD_LEN: usize = 100;
+            const MAX_BAD_COUNTER: usize = 30;
+            let this_frame_len = frames[0].data.len();
+            if this_frame_len < MIN_BAD_LEN && this_frame_len == self.last_frame_len {
+                self.same_bad_len_counter += 1;
+                if self.same_bad_len_counter >= MAX_BAD_COUNTER {
+                    log::info!(
+                        "{} times encoding len is {}, switch",
+                        self.same_bad_len_counter,
+                        self.last_frame_len
+                    );
+                    bail!(crate::codec::ENCODE_NEED_SWITCH);
+                }
+            } else {
+                self.same_bad_len_counter = 0;
+            }
+            self.last_frame_len = this_frame_len;
+            let frames = EncodedVideoFrames {
+                frames: frames.into(),
+                ..Default::default()
+            };
+            match self.format {
+                DataFormat::H264 => vf.set_h264s(frames),
+                DataFormat::H265 => vf.set_h265s(frames),
+                _ => bail!("{:?} not supported", self.format),
+            }
+            Ok(vf)
+        } else {
+            Err(anyhow!("no valid frame"))
+        }
+    }
+
+    fn yuvfmt(&self) -> EncodeYuvFormat {
+        // useless
+        EncodeYuvFormat {
+            pixfmt: Pixfmt::BGRA,
+            w: self.ctx.d.width as _,
+            h: self.ctx.d.height as _,
+            stride: Vec::new(),
+            u: 0,
+            v: 0,
+        }
+    }
+
+    #[cfg(feature = "vram")]
+    fn input_texture(&self) -> bool {
+        true
+    }
+
+    fn set_quality(&mut self, ratio: f32) -> ResultType<()> {
+        let bitrate = Self::bitrate(
+            self.ctx.f.data_format,
+            self.ctx.d.width as _,
+            self.ctx.d.height as _,
+            ratio,
+        );
+        if bitrate > 0 {
+            if self.encoder.set_bitrate((bitrate) as _).is_ok() {
+                self.bitrate = bitrate;
+            }
+        }
+        Ok(())
+    }
+
+    fn bitrate(&self) -> u32 {
+        self.bitrate
+    }
+
+    fn support_changing_quality(&self) -> bool {
+        true
+    }
+
+    fn latency_free(&self) -> bool {
+        true
+    }
+
+    fn is_hardware(&self) -> bool {
+        true
+    }
+
+    fn disable(&self) {
+        HwCodecConfig::clear(true, true);
+    }
+}
+
+impl VRamEncoder {
+    pub fn try_get(device: &AdapterDevice, format: CodecFormat) -> Option<FeatureContext> {
+        let v: Vec<_> = Self::available(format)
+            .drain(..)
+            .filter(|e| e.luid == device.luid)
+            .collect();
+        if v.len() > 0 {
+            // prefer ffmpeg
+            if let Some(ctx) = v.iter().find(|c| c.driver == Driver::FFMPEG) {
+                return Some(ctx.clone());
+            }
+            Some(v[0].clone())
+        } else {
+            None
+        }
+    }
+
+    pub fn available(format: CodecFormat) -> Vec<FeatureContext> {
+        let fallbacks = FALLBACK_GDI_DISPLAYS.lock().unwrap().clone();
+        if !fallbacks.is_empty() {
+            log::info!("fallback gdi displays not empty: {fallbacks:?}");
+            return vec![];
+        }
+        let not_use = ENOCDE_NOT_USE.lock().unwrap().clone();
+        if not_use.values().any(|not_use| *not_use) {
+            log::info!("currently not use vram encoders: {not_use:?}");
+            return vec![];
+        }
+        let data_format = match format {
+            CodecFormat::H264 => DataFormat::H264,
+            CodecFormat::H265 => DataFormat::H265,
+            _ => return vec![],
+        };
+        let v: Vec<_> = crate::hwcodec::HwCodecConfig::get()
+            .vram_encode
+            .drain(..)
+            .filter(|c| c.data_format == data_format)
+            .collect();
+        if crate::hwcodec::HwRamEncoder::try_get(format).is_some() {
+            // has fallback, no need to require all adapters support
+            v
+        } else {
+            let Ok(displays) = crate::Display::all() else {
+                log::error!("failed to get displays");
+                return vec![];
+            };
+            if displays.is_empty() {
+                log::error!("no display found");
+                return vec![];
+            }
+            let luids = displays
+                .iter()
+                .map(|d| d.adapter_luid())
+                .collect::<Vec<_>>();
+            if luids
+                .iter()
+                .all(|luid| v.iter().any(|f| Some(f.luid) == *luid))
+            {
+                v
+            } else {
+                log::info!("not all adapters support {data_format:?}, luids = {luids:?}");
+                vec![]
+            }
+        }
+    }
+
+    pub fn encode(&mut self, texture: *mut c_void, ms: i64) -> ResultType<Vec<EncodeFrame>> {
+        match self.encoder.encode(texture, ms) {
+            Ok(v) => {
+                let mut data = Vec::<EncodeFrame>::new();
+                data.append(v);
+                Ok(data)
+            }
+            Err(_) => Ok(Vec::<EncodeFrame>::new()),
+        }
+    }
+
+    pub fn bitrate(fmt: DataFormat, width: usize, height: usize, ratio: f32) -> u32 {
+        crate::hwcodec::HwRamEncoder::calc_bitrate(width, height, ratio, fmt == DataFormat::H264)
+    }
+
+    pub fn set_not_use(video_service_name: String, not_use: bool) {
+        log::info!("set {video_service_name} not use vram encode to {not_use}");
+        ENOCDE_NOT_USE
+            .lock()
+            .unwrap()
+            .insert(video_service_name, not_use);
+    }
+
+    pub fn set_fallback_gdi(video_service_name: String, fallback: bool) {
+        if fallback {
+            FALLBACK_GDI_DISPLAYS
+                .lock()
+                .unwrap()
+                .insert(video_service_name);
+        } else {
+            FALLBACK_GDI_DISPLAYS
+                .lock()
+                .unwrap()
+                .remove(&video_service_name);
+        }
+    }
+}
+
+pub struct VRamDecoder {
+    decoder: Decoder,
+}
+
+impl VRamDecoder {
+    pub fn try_get(format: CodecFormat, luid: Option<i64>) -> Option<DecodeContext> {
+        let v: Vec<_> = Self::available(format, luid);
+        if v.len() > 0 {
+            // prefer ffmpeg
+            if let Some(ctx) = v.iter().find(|c| c.driver == Driver::FFMPEG) {
+                return Some(ctx.clone());
+            }
+            Some(v[0].clone())
+        } else {
+            None
+        }
+    }
+
+    pub fn available(format: CodecFormat, luid: Option<i64>) -> Vec<DecodeContext> {
+        let luid = luid.unwrap_or_default();
+        let data_format = match format {
+            CodecFormat::H264 => DataFormat::H264,
+            CodecFormat::H265 => DataFormat::H265,
+            _ => return vec![],
+        };
+        crate::hwcodec::HwCodecConfig::get()
+            .vram_decode
+            .drain(..)
+            .filter(|c| c.data_format == data_format && c.luid == luid && luid != 0)
+            .collect()
+    }
+
+    pub fn possible_available_without_check() -> (bool, bool) {
+        if !enable_vram_option(false) {
+            return (false, false);
+        }
+        let v = crate::hwcodec::HwCodecConfig::get().vram_decode;
+        (
+            v.iter().any(|d| d.data_format == DataFormat::H264),
+            v.iter().any(|d| d.data_format == DataFormat::H265),
+        )
+    }
+
+    pub fn new(format: CodecFormat, luid: Option<i64>) -> ResultType<Self> {
+        let ctx = Self::try_get(format, luid).ok_or(anyhow!("Failed to get decode context"))?;
+        log::info!("try create vram decoder: {ctx:?}");
+        match Decoder::new(ctx) {
+            Ok(decoder) => Ok(Self { decoder }),
+            Err(_) => {
+                HwCodecConfig::clear(true, false);
+                Err(anyhow!(format!(
+                    "Failed to create decoder, format: {:?}",
+                    format
+                )))
+            }
+        }
+    }
+    pub fn decode<'a>(&'a mut self, data: &[u8]) -> ResultType<Vec<VRamDecoderImage<'a>>> {
+        match self.decoder.decode(data) {
+            Ok(v) => Ok(v.iter().map(|f| VRamDecoderImage { frame: f }).collect()),
+            Err(e) => Err(anyhow!(e)),
+        }
+    }
+}
+
+pub struct VRamDecoderImage<'a> {
+    pub frame: &'a DecodeFrame,
+}
+
+impl VRamDecoderImage<'_> {}
+
+pub(crate) fn check_available_vram() -> (Vec<FeatureContext>, Vec<DecodeContext>, String) {
+    let d = DynamicContext {
+        device: None,
+        width: 1280,
+        height: 720,
+        kbitrate: 5000,
+        framerate: 60,
+        gop: MAX_GOP as _,
+    };
+    let encoders = encode::available(d);
+    let decoders = decode::available();
+    let available = Available {
+        e: encoders.clone(),
+        d: decoders.clone(),
+    };
+    (
+        encoders,
+        decoders,
+        available.serialize().unwrap_or_default(),
+    )
+}

libs/scrap/src/common/wayland.rs

@@ -0,0 +1,129 @@
+use crate::{
+    wayland::{capturable::*, *},
+    Frame, TraitCapturer,
+};
+use std::{io, sync::RwLock, time::Duration};
+
+use super::x11::PixelBuffer;
+
+pub struct Capturer(Display, Box<dyn Recorder>, Vec<u8>);
+
+lazy_static::lazy_static! {
+    static ref MAP_ERR: RwLock<Option<fn(err: String)-> io::Error>> = Default::default();
+}
+
+pub fn set_map_err(f: fn(err: String) -> io::Error) {
+    *MAP_ERR.write().unwrap() = Some(f);
+}
+
+fn map_err<E: ToString>(err: E) -> io::Error {
+    if let Some(f) = *MAP_ERR.read().unwrap() {
+        f(err.to_string())
+    } else {
+        io::Error::new(io::ErrorKind::Other, err.to_string())
+    }
+}
+
+impl Capturer {
+    pub fn new(display: Display) -> io::Result<Capturer> {
+        let r = display.0.recorder(false).map_err(map_err)?;
+        Ok(Capturer(display, r, Default::default()))
+    }
+
+    pub fn width(&self) -> usize {
+        self.0.width()
+    }
+
+    pub fn height(&self) -> usize {
+        self.0.height()
+    }
+}
+
+impl TraitCapturer for Capturer {
+    fn frame<'a>(&'a mut self, timeout: Duration) -> io::Result<Frame<'a>> {
+        match self.1.capture(timeout.as_millis() as _).map_err(map_err)? {
+            PixelProvider::BGR0(w, h, x) => Ok(Frame::PixelBuffer(PixelBuffer::new(
+                x,
+                crate::Pixfmt::BGRA,
+                w,
+                h,
+            ))),
+            PixelProvider::RGB0(w, h, x) => Ok(Frame::PixelBuffer(PixelBuffer::new(
+                x,
+                crate::Pixfmt::RGBA,
+                w,
+                h,
+            ))),
+            PixelProvider::NONE => Err(std::io::ErrorKind::WouldBlock.into()),
+            _ => Err(map_err("Invalid data")),
+        }
+    }
+}
+
+pub struct Display(pub(crate) pipewire::PipeWireCapturable);
+
+impl Display {
+    pub fn primary() -> io::Result<Display> {
+        let mut all = Display::all()?;
+        if all.is_empty() {
+            return Err(io::ErrorKind::NotFound.into());
+        }
+        Ok(all.remove(0))
+    }
+
+    pub fn all() -> io::Result<Vec<Display>> {
+        Ok(pipewire::get_capturables()
+            .map_err(map_err)?
+            .drain(..)
+            .map(|x| Display(x))
+            .collect())
+    }
+
+    pub fn width(&self) -> usize {
+        self.physical_width()
+    }
+
+    pub fn height(&self) -> usize {
+        self.physical_height()
+    }
+
+    pub fn physical_width(&self) -> usize {
+        self.0.physical_size.0
+    }
+
+    pub fn physical_height(&self) -> usize {
+        self.0.physical_size.1
+    }
+
+    pub fn logical_width(&self) -> usize {
+        self.0.logical_size.0
+    }
+
+    pub fn logical_height(&self) -> usize {
+        self.0.logical_size.1
+    }
+
+    pub fn scale(&self) -> f64 {
+        if self.logical_width() == 0 {
+            1.0
+        } else {
+            self.physical_width() as f64 / self.logical_width() as f64
+        }
+    }
+
+    pub fn origin(&self) -> (i32, i32) {
+        self.0.position
+    }
+
+    pub fn is_online(&self) -> bool {
+        true
+    }
+
+    pub fn is_primary(&self) -> bool {
+        self.0.primary
+    }
+
+    pub fn name(&self) -> String {
+        "".to_owned()
+    }
+}

libs/scrap/src/common/x11.rs

@@ -0,0 +1,139 @@
+use crate::{common::TraitCapturer, x11, Frame, Pixfmt, TraitPixelBuffer};
+use std::{io, time::Duration};
+
+pub struct Capturer(x11::Capturer);
+
+pub const IS_CURSOR_EMBEDDED: bool = false;
+
+impl Capturer {
+    pub fn new(display: Display) -> io::Result<Capturer> {
+        x11::Capturer::new(display.0).map(Capturer)
+    }
+
+    pub fn width(&self) -> usize {
+        self.0.display().rect().w as usize
+    }
+
+    pub fn height(&self) -> usize {
+        self.0.display().rect().h as usize
+    }
+}
+
+impl TraitCapturer for Capturer {
+    fn frame<'a>(&'a mut self, _timeout: Duration) -> io::Result<Frame<'a>> {
+        let width = self.width();
+        let height = self.height();
+        let pixfmt = self.0.display().pixfmt();
+        Ok(Frame::PixelBuffer(PixelBuffer::new(
+            self.0.frame()?,
+            pixfmt,
+            width,
+            height,
+        )))
+    }
+}
+
+pub struct PixelBuffer<'a> {
+    data: &'a [u8],
+    pixfmt: Pixfmt,
+    width: usize,
+    height: usize,
+    stride: Vec<usize>,
+}
+
+impl<'a> PixelBuffer<'a> {
+    pub fn new(data: &'a [u8], pixfmt: Pixfmt, width: usize, height: usize) -> Self {
+        let stride0 = data.len() / height;
+        let mut stride = Vec::new();
+        stride.push(stride0);
+        Self {
+            data,
+            pixfmt,
+            width,
+            height,
+            stride,
+        }
+    }
+}
+
+impl<'a> TraitPixelBuffer for PixelBuffer<'a> {
+    fn data(&self) -> &[u8] {
+        self.data
+    }
+
+    fn width(&self) -> usize {
+        self.width
+    }
+
+    fn height(&self) -> usize {
+        self.height
+    }
+
+    fn stride(&self) -> Vec<usize> {
+        self.stride.clone()
+    }
+
+    fn pixfmt(&self) -> crate::Pixfmt {
+        self.pixfmt
+    }
+}
+
+pub struct Display(x11::Display);
+
+impl Display {
+    pub fn primary() -> io::Result<Display> {
+        let server = match x11::Server::default() {
+            Ok(server) => server,
+            Err(_) => return Err(io::ErrorKind::ConnectionRefused.into()),
+        };
+
+        let mut displays = x11::Server::displays(server);
+        let mut best = displays.next();
+        if best.as_ref().map(|x| x.is_default()) == Some(false) {
+            best = displays.find(|x| x.is_default()).or(best);
+        }
+
+        match best {
+            Some(best) => Ok(Display(best)),
+            None => Err(io::ErrorKind::NotFound.into()),
+        }
+    }
+
+    pub fn all() -> io::Result<Vec<Display>> {
+        let server = match x11::Server::default() {
+            Ok(server) => server,
+            Err(_) => return Err(io::ErrorKind::ConnectionRefused.into()),
+        };
+
+        Ok(x11::Server::displays(server).map(Display).collect())
+    }
+
+    pub fn width(&self) -> usize {
+        self.0.rect().w as usize
+    }
+
+    pub fn height(&self) -> usize {
+        self.0.rect().h as usize
+    }
+
+    pub fn origin(&self) -> (i32, i32) {
+        let r = self.0.rect();
+        (r.x as _, r.y as _)
+    }
+
+    pub fn is_online(&self) -> bool {
+        true
+    }
+
+    pub fn is_primary(&self) -> bool {
+        self.0.is_default()
+    }
+
+    pub fn name(&self) -> String {
+        self.0.name()
+    }
+
+    pub fn get_shm_status(&self) -> Result<(), x11::Error> {
+        self.0.server().get_shm_status()
+    }
+}