HarmonyOS鸿蒙Next中基于ImageEffect为相机预览添加滤镜

HarmonyOS鸿蒙Next中基于ImageEffect为相机预览添加滤镜

场景描述：

使用系统ImageEffect提供的图片编辑能力，模拟系统相机滤镜功能，实现自定义相机预览过程中动态开启关闭的滤镜效果。

其中滤镜效果分为两类： 1、系统提供的滤镜（亮度、对比度、裁剪等）。 2、自定义滤镜效果（例如黑白滤镜）。

效果展示：

方案描述：

1、自定义相机预览场景中XComponent组件为相机预览流提供SurfaceId，调用相机初始化的napi接口传入到native侧。 2、创建ImageEffect对象，向滤镜链路中添加系统自带的滤镜以及自定义滤镜。 3、在native c++层将SurfaceId转换成OHNativeWindow，并调OH_ImageEffect_SetOutputSurface设置输出显示的OHNativeWindow。 4、从ImageEffect中获取输入的OHNativeWindow，再从OHNativeWindow中获取到新的SurfaceId。 5、用获取到的新SurfaceId创建相机预览流，完成将数据链路由相机->XComponent改为相机->ImageEffect 滤镜链路 -> XComponent。

步骤及关键代码：

1、ArkTS侧从XComponent中获取surfaceId，传入native侧：

XComponent(this.options) 
  .onLoad(async () => { 
    Logger.info(TAG, 'onLoad is called'); 
    this.surfaceId = this.mXComponentController.getXComponentSurfaceId(); 
    Logger.info(TAG, `onLoad surfaceId: ${this.surfaceId}`); 
    Logger.info(TAG, 'initCamera start'); 
    cameraNapi.initCamera(this.surfaceId, this.settingDataObj.focusMode, this.cameraDeviceIndex); 
    Logger.info(TAG, 'initCamera end'); 
  })

2、创建ImageEffect对象，并添加滤镜到滤镜链中：

imageEffect = OH_ImageEffect_Create("imageEdit");

a. 添加系统自带的亮度滤镜：

static void AddSystemFilter() { 
    OH_EffectFilter *filter = OH_ImageEffect_AddFilter(imageEffect, OH_EFFECT_BRIGHTNESS_FILTER); 
    // 设置滤镜参数, 例如：滤镜强度设置为50。 
    ImageEffect_Any value = {.dataType = ImageEffect_DataType::EFFECT_DATA_TYPE_FLOAT, .dataValue.floatValue = 50.f}; 
    OH_EffectFilter_SetValue(filter, OH_EFFECT_FILTER_INTENSITY_KEY, &value); 
}

b. 添加自定义滤镜：

bool OnApplyRGBA8888(OH_EffectBufferInfo *src) { 
    void *addr = nullptr; 
    OH_EffectBufferInfo_GetAddr(src, &addr); 
    auto *srcRgb = (unsigned char *)addr; 

    int32_t width = 0; 
    OH_EffectBufferInfo_GetWidth(src, &width); 
    int32_t height = 0; 
    OH_EffectBufferInfo_GetHeight(src, &height); 
    int32_t rowStride = 0; 
    OH_EffectBufferInfo_GetRowSize(src, &rowStride); 

    for (int y = 0; y < height; ++y) { 
        for (int x = 0; x < width; ++x) { 
            for (int i = 0; i < 4; ++i) { 
                uint32_t index = rowStride * y + x * sizeof(int) + i; 
                srcRgb[index] = (i == 3) ? 255 : srcRgb[index]; 
            } 
        } 
    } 
    return true; 
} 

bool OnApplyYUVNV(OH_EffectBufferInfo *src) { 
    void *buffer = nullptr; 
    OH_EffectBufferInfo_GetAddr(src, &buffer); 
    int32_t width = 0; 
    OH_EffectBufferInfo_GetWidth(src, &width); 
    int32_t height = 0; 
    OH_EffectBufferInfo_GetHeight(src, &height); 
    int32_t rowStride = 0; 
    OH_EffectBufferInfo_GetRowSize(src, &rowStride); 
    memset((unsigned char *)buffer + rowStride * height, 0, rowStride * height / 2); 
    return true; 
} 

static bool Apply(OH_EffectFilter *filter, OH_EffectBufferInfo *src, OH_EffectFilterDelegate_PushData pushData) 
{ 
    ImageEffect_Format format = ImageEffect_Format::EFFECT_PIXEL_FORMAT_UNKNOWN; 
    OH_EffectBufferInfo_GetEffectFormat(src, &format); 
    bool result = true; 
    switch (format) { 
    case ImageEffect_Format::EFFECT_PIXEL_FORMAT_RGBA8888: 
        result = OnApplyRGBA8888(src); 
        break; 
    case ImageEffect_Format::EFFECT_PIXEL_FORMAT_NV12: 
    case ImageEffect_Format::EFFECT_PIXEL_FORMAT_NV21: 
        result = OnApplyYUVNV(src); 
        break; 
    default: 
        LOG_E("format not support! format=%d", format); 
        result = false; 
        break; 
    } 
    pushData(filter, src); 
    return result; 
} 

static OH_EffectFilter *Restore(const char *info) 
{ 
    return nullptr; 
} 

static void AddCustomFilter() { 
    // 自定义算子能力信息 
    OH_EffectFilterInfo *filterInfo = OH_EffectFilterInfo_Create(); 
    OH_EffectFilterInfo_SetFilterName(filterInfo, "CustomEFilter"); 
    ImageEffect_BufferType bufferType = ImageEffect_BufferType::EFFECT_BUFFER_TYPE_PIXEL; 
    OH_EffectFilterInfo_SetSupportedBufferTypes(filterInfo, sizeof(bufferType) / sizeof(ImageEffect_BufferType), 
                                                &bufferType); 
    ImageEffect_Format format[] = { 
        ImageEffect_Format::EFFECT_PIXEL_FORMAT_RGBA8888, 
        ImageEffect_Format::EFFECT_PIXEL_FORMAT_NV12, 
        ImageEffect_Format::EFFECT_PIXEL_FORMAT_NV21, 
    }; 
    OH_EffectFilterInfo_SetSupportedFormats(filterInfo, sizeof(format) / sizeof(ImageEffect_Format), format); 
    // 自定义算子实现接口 
    delegate = { 
        .setValue = [](OH_EffectFilter *filter, const char *key, const ImageEffect_Any *value) { return true; }, 
        .render = [](OH_EffectFilter *filter, OH_EffectBufferInfo *src, OH_EffectFilterDelegate_PushData pushData) { 
            return Apply(filter, src, pushData); 
        }, 
        .save = [](OH_EffectFilter *filter, char **info) { return true; }, 
        .restore = [](const char *info) { return Restore(info); }}; 
    OH_EffectFilter_Register(filterInfo, &delegate); 
    OH_EffectFilterInfo_Release(filterInfo); 
    OH_ImageEffect_AddFilter(imageEffect, "CustomEFilter"); 
}

3、native侧获取XComponent组件提供的surfaceId，并调用ImageEffect接口获取新的surfaceId：

static napi_value GetImageEffectSurfaceId(napi_env env, napi_callback_info info) { 
    size_t argc = 1; 
    napi_value args[1] = {nullptr}; 
    napi_value result; 
    size_t surfaceIdLen = 0; 
    char *xComponentSurfaceId = nullptr; 

    napi_get_cb_info(env, info, &argc, args, nullptr, nullptr); 
    napi_get_value_string_utf8(env, args[0], nullptr, 0, &surfaceIdLen); 
    xComponentSurfaceId = new char[surfaceIdLen + 1]; 
    napi_get_value_string_utf8(env, args[0], xComponentSurfaceId, surfaceIdLen + 1, &surfaceIdLen); 

    ImageEffect_ErrorCode errorCode; 
    imageEffect = OH_ImageEffect_Create("effectDemo"); 
    ImageEffect_Any runningType{.dataType = ImageEffect_DataType::EFFECT_DATA_TYPE_INT32, .dataValue.int32Value = 2}; 
    errorCode = OH_ImageEffect_Configure(imageEffect, "runningType", &runningType); 

    // 添加滤镜，获取 OH_EffectFilter 实例。多次调用该接口可以添加多个滤镜，组成滤镜链。 
    AddSystemFilter(); 
    AddCustomFilter(); 

    // 根据SurfaceId创建NativeWindow，注意创建出来的NativeWindow在使用结束后需要主动调用OH_NativeWindow_DestoryNativeWindow进行释放。 
    uint64_t outputSurfaceId; 
    istrstream iss(xComponentSurfaceId); 
    iss >> outputSurfaceId; 
    OHNativeWindow *outputNativeWindow = nullptr; 
    OH_NativeWindow_CreateNativeWindowFromSurfaceId(outputSurfaceId, &outputNativeWindow); 

    // 设置输出显示的Surface。 
    errorCode = OH_ImageEffect_SetOutputSurface(imageEffect, outputNativeWindow); 
    ReleaseNativeWindow(outputNativeWindow); 

    // 获取输入的Surface。注意获取的inputNativeWindow在使用结束后需要主动调用OH_NativeWindow_DestoryNativeWindow进行释放。 
    OHNativeWindow *inputNativeWindow = nullptr; 
    errorCode = OH_ImageEffect_GetInputSurface(imageEffect, &inputNativeWindow); 

    // 从获取到输入的NativeWindow中获取SurfaceId。 
    uint64_t inputSurfaceId = 0; 
    OH_NativeWindow_GetSurfaceId(inputNativeWindow, &inputSurfaceId); 
    ReleaseNativeWindow(inputNativeWindow); 

    // 将SurfaceId转成字符串进行返回。 
    string inputSurfaceIdStr = to_string(inputSurfaceId); 
    napi_create_string_utf8(env, inputSurfaceIdStr.c_str(), inputSurfaceIdStr.length(), &result); 

    return result; 
}

4、将获取到的SurfaceId通过构造函数传递给相机框架，调用相机接口启动预览：

class NDKCamera {
  NDKCamera(char *str, uint32_t focusMode, uint32_t cameraDeviceIndex)
       : previewSurfaceId_(str), cameras_(nullptr), focusMode_(focusMode), cameraDeviceIndex_(cameraDeviceIndex),
         cameraOutputCapability_(nullptr), cameraInput_(nullptr), captureSession_(nullptr), size_(0),
         isCameraMuted_(nullptr), profile_(nullptr), photoSurfaceId_(nullptr), previewOutput_(nullptr),
         photoOutput_(nullptr), metaDataObjectType_(nullptr), metadataOutput_(nullptr), isExposureModeSupported_(false),
         isFocusModeSupported_(false), exposureMode_(EXPOSURE_MODE_LOCKED), minExposureBias_(0), maxExposureBias_(0),
         step_(0), ret_(CAMERA_OK) { 
       valid_ = false;
       ReleaseCamera();
       Camera_ErrorCode ret = OH_Camera_GetCameraManager(&cameraManager_);
       if (cameraManager_ == nullptr || ret != CAMERA_OK) {
           OH_LOG_ERROR(LOG_APP, "Get CameraManager failed.");
       }

       ret = OH_CameraManager_CreateCaptureSession(cameraManager_, &captureSession_);
       if (captureSession_ == nullptr || ret != CAMERA_OK) {
           OH_LOG_ERROR(LOG_APP, "Create captureSession failed.");
       }
       CaptureSessionRegisterCallback();
       GetSupportedCameras();
       GetSupportedOutputCapability();
       CreatePreviewOutput();
       CreateCameraInput();
       CameraInputOpen();
       CameraManagerRegisterCallback();
       SessionFlowFn();
       valid_ = true;
   }

  Camera_ErrorCode NDKCamera::CreatePreviewOutput(void) { 
    profile_ = cameraOutputCapability_->previewProfiles[0];
    if (profile_ == nullptr) {
        OH_LOG_ERROR(LOG_APP, "Get previewProfiles failed.");
        return CAMERA_INVALID_ARGUMENT;
    }
    ret_ = OH_CameraManager_CreatePreviewOutput(cameraManager_, profile_, previewSurfaceId_, &previewOutput_);
    if (previewSurfaceId_ == nullptr || previewOutput_ == nullptr || ret_ != CAMERA_OK) {
        OH_LOG_ERROR(LOG_APP, "CreatePreviewOutput failed.");
        return CAMERA_INVALID_ARGUMENT;
    }
    return ret_;
}

5、点击开启/关闭按钮，进行滤镜效果的开启和关闭：

Image($r('app.media.camera_filter'))
  .width(px2vp(Constants.ICON_SIZE))
  .height(px2vp(Constants.ICON_SIZE))
  .onClick(async () => {
    if (!this.filterIsOpen) {
      cameraDemo.startImageEffect();
    } else {
      cameraDemo.stopImageEffect();
    }
    this.filterIsOpen = !this.filterIsOpen;
  })

static napi_value StartImageEffect(napi_env env, napi_callback_info info) { 
    napi_value result; 
    ImageEffect_ErrorCode errCode = OH_ImageEffect_Start(imageEffect); 
    int ret = errCode != ImageEffect_ErrorCode::EFFECT_SUCCESS ? -1 : 0; 
    napi_create_int32(env, ret, &result); 
    return result; 
} 

static napi_value StopImageEffect(napi_env env, napi_callback_info info) { 
    napi_value result; 
    ImageEffect_ErrorCode errCode = OH_ImageEffect_Stop(imageEffect); 
    int ret = errCode != ImageEffect_ErrorCode::EFFECT_SUCCESS ? -1 : 0; 
    napi_create_int32(env, ret, &result); 
    return result; 
}

注意事项

在使用提亮滤镜和自定义滤镜的过程中，经验证发现： 1、在滤镜链中仅加入提亮滤镜时，需要使用如下代码配置ImageEffect的runningtype，使其通过cpu执行，否则会在相机预览surface模式下，画面卡死：

OH_EffectFilter *filter = OH_ImageEffect_AddFilter(imageEffect, OH_EFFECT_BRIGHTNESS_FILTER);

2、在滤镜链中同时存在自定义滤镜和提亮滤镜的时候，不进行上述配置，先添加提亮滤镜，再添加自定义滤镜，画面卡死。只有先添加自定义滤镜，再添加提亮滤镜，画面正常：

ImageEffect_Any runningType{.dataType = ImageEffect_DataType::EFFECT_DATA_TYPE_INT32, .dataValue.int32Value = 2};      
errorCode = OH_ImageEffect_Configure(imageEffect, "runningType", &runningType);