-
-
Notifications
You must be signed in to change notification settings - Fork 2.3k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Draft: Add SKSLEffect. #17981
base: master
Are you sure you want to change the base?
Draft: Add SKSLEffect. #17981
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
<UserControl xmlns="https://github.com/avaloniaui" | ||
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" | ||
xmlns:d="http://schemas.microsoft.com/expression/blend/2008" | ||
xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006" | ||
mc:Ignorable="d" d:DesignWidth="800" d:DesignHeight="450" | ||
x:Class="ControlCatalog.Pages.SKSLEffectPage"> | ||
<UniformGrid Columns="3" Height="200"> | ||
<Rectangle x:Name="Rectangle0" Fill="Red" Margin="20"/> | ||
<Rectangle x:Name="Rectangle1" Fill="Red" Margin="20"/> | ||
<Rectangle x:Name="Rectangle2" Fill="Red" Margin="20"/> | ||
</UniformGrid> | ||
</UserControl> |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,185 @@ | ||
using System; | ||
using System.Collections.Generic; | ||
using System.Diagnostics; | ||
using System.Text; | ||
using Avalonia; | ||
using Avalonia.Animation; | ||
using Avalonia.Controls; | ||
using Avalonia.Controls.Shapes; | ||
using Avalonia.Data; | ||
using Avalonia.Markup.Xaml; | ||
using Avalonia.Media; | ||
using Avalonia.Platform; | ||
using Avalonia.Skia.Effects; | ||
using Avalonia.Styling; | ||
using SkiaSharp; | ||
|
||
namespace ControlCatalog.Pages | ||
{ | ||
public partial class SKSLEffectPage : UserControl | ||
{ | ||
public SKSLEffectPage() | ||
{ | ||
InitializeComponent(); | ||
InitEffect(); | ||
} | ||
|
||
private void InitializeComponent() | ||
{ | ||
AvaloniaXamlLoader.Load(this); | ||
} | ||
|
||
private void InitEffect() | ||
{ | ||
var rectangle1 = this.FindControl<Rectangle>("Rectangle1")!; | ||
var shaderBuilder = CreateSimpleShaderBuilder(); | ||
if (shaderBuilder != null) | ||
{ | ||
rectangle1.Effect = new SKSLEffect(shaderBuilder) | ||
{ | ||
ChildShaderNames = ["src"], | ||
Inputs = [null], | ||
}; | ||
} | ||
|
||
var rectangle2 = this.FindControl<Rectangle>("Rectangle2")!; | ||
try | ||
{ | ||
var effect = new DissolveSKSLEffect(); | ||
effect.Progress = 0.5f; | ||
effect[!DissolveSKSLEffect.ResolutionProperty] = new Binding | ||
{ | ||
Source = rectangle2, | ||
Path = "Bounds.Size", | ||
}; | ||
rectangle2.Effect = effect; | ||
|
||
var animation = new Animation | ||
{ | ||
Duration = TimeSpan.FromSeconds(2), | ||
IterationCount = IterationCount.Infinite, | ||
PlaybackDirection = PlaybackDirection.Alternate, | ||
Children = { | ||
new KeyFrame | ||
{ | ||
Setters = | ||
{ | ||
new Setter(DissolveSKSLEffect.ProgressProperty, 0f), | ||
}, | ||
KeyTime = TimeSpan.FromSeconds(0), | ||
}, | ||
new KeyFrame | ||
{ | ||
Setters = | ||
{ | ||
new Setter(DissolveSKSLEffect.ProgressProperty, 1f), | ||
}, | ||
KeyTime = TimeSpan.FromSeconds(2), | ||
} | ||
} | ||
}; | ||
|
||
_ = animation.RunAsync(effect); | ||
} | ||
catch | ||
{ | ||
// Do not crash. | ||
} | ||
} | ||
|
||
private SKRuntimeShaderBuilder? CreateSimpleShaderBuilder() | ||
{ | ||
var sksl = @" | ||
uniform shader src; | ||
|
||
float4 main(float2 coord) { | ||
return src.eval(coord).bgra; | ||
} | ||
"; | ||
var effect = SKRuntimeEffect.CreateShader(sksl, out var str); | ||
if (effect != null) | ||
{ | ||
return new SKRuntimeShaderBuilder(effect); | ||
} | ||
else | ||
{ | ||
return null; | ||
} | ||
} | ||
} | ||
|
||
public class DissolveSKSLEffect : SKSLEffect | ||
{ | ||
public static readonly StyledProperty<float> ProgressProperty = AvaloniaProperty.Register<DissolveSKSLEffect, float>(nameof(Progress), default); | ||
|
||
public float Progress | ||
{ | ||
get => GetValue(ProgressProperty); | ||
set => SetValue(ProgressProperty, value); | ||
} | ||
|
||
public static readonly StyledProperty<Size> ResolutionProperty = AvaloniaProperty.Register<DissolveSKSLEffect, Size>(nameof(Resolution), default); | ||
|
||
public Size Resolution | ||
{ | ||
get => GetValue(ResolutionProperty); | ||
set => SetValue(ResolutionProperty, value); | ||
} | ||
|
||
public DissolveSKSLEffect() : base(CreateShaderBuilder()) | ||
{ | ||
ChildShaderNames = ["src"]; | ||
Inputs = [null]; | ||
AffectsRender<SKSLEffect>(ProgressProperty); | ||
|
||
RegisterUniform("progress", ProgressProperty); | ||
RegisterUniform("resolution", ResolutionProperty); | ||
} | ||
|
||
private static SKRuntimeShaderBuilder? s_shaderBuilder; | ||
private static SKRuntimeShaderBuilder CreateShaderBuilder() | ||
{ | ||
if (s_shaderBuilder != null) | ||
{ | ||
return s_shaderBuilder; | ||
} | ||
|
||
var sksl = @" | ||
uniform float2 resolution; | ||
uniform shader src; | ||
uniform shader noise; | ||
uniform float2 noiseResolution; | ||
uniform float progress; | ||
|
||
float4 main(float2 coord) { | ||
float val = noise.eval(fract(coord / resolution) * noiseResolution).x; | ||
|
||
if(val < progress) | ||
{ | ||
return src.eval(coord); | ||
} | ||
else | ||
{ | ||
return float4(0,0,0,0); | ||
} | ||
} | ||
"; | ||
var effect = SKRuntimeEffect.CreateShader(sksl, out var str); | ||
if (effect != null) | ||
{ | ||
var noise = AssetLoader.Open(new Uri("avares://ControlCatalog/Assets/noise.png")); | ||
var noiseImage = SKImage.FromEncodedData(noise); | ||
var noiseImageShader = SKShader.CreateImage(noiseImage); | ||
var builder = new SKRuntimeShaderBuilder(effect); | ||
builder.Uniforms["noiseResolution"] = new SKSize(noiseImage.Width, noiseImage.Height); | ||
builder.Children["noise"] = noiseImageShader; | ||
s_shaderBuilder = builder; | ||
return s_shaderBuilder; | ||
} | ||
else | ||
{ | ||
throw new NotSupportedException(); | ||
} | ||
} | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
using System; | ||
using System.Collections.Generic; | ||
using System.Linq; | ||
using System.Text; | ||
using System.Threading.Tasks; | ||
|
||
namespace Avalonia.Media | ||
{ | ||
public interface IShaderEffect : IEffect | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Custom effects need to report their expected padding, i. e. the area outside of the affected drawing bounds they will alter (e. g. drop-shadow(5px 5px) would add (0,0,5,5) padding for renderer invalidation). Note that this can depend on uniform values. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think shader effect shouldn't has padding? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. SKImageFilter can have padding, that's how our blur and drop shadow effects work. They report padding to the compositor and it extends the intermediate texture bounds. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Actually, right now the intermediate texture size always matches the render target size. That's something we've planned to change but never actually implemented. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
But runtimeshader seems to be not? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The "padding" is just shader sampling the source at a different coordinate than it's currently producing output for. E. g. a simple implementation of There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
I have read skia source and usage of
And i think runtimeshader cannot affect. It's just a fragment shader and a subset of There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. In WPF it's a normal thing for ShaderEffect to extend the affected Visual's render bounds. We should do the same. If a custom SKImageFilter somehow doesn't support that, we need to manually manage the intermediate texture and use SKShader instead of SKImageFilter. |
||
{ | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,94 @@ | ||
using System.Collections.Generic; | ||
using Avalonia.Logging; | ||
using Avalonia.Media; | ||
using SkiaSharp; | ||
|
||
namespace Avalonia.Skia.Effects | ||
{ | ||
public class SKSLEffect : Effect, ISKSLEffect, IMutableEffect | ||
{ | ||
public SKRuntimeShaderBuilder ShaderBuilder { get; set; } | ||
|
||
public float MaxSampleRadius { get; set; } = 0f; | ||
|
||
public string[] ChildShaderNames { get; set; } = []; | ||
|
||
public SKImageFilter?[] Inputs { get; set; } = []; | ||
|
||
private readonly Dictionary<string, AvaloniaProperty> _uniformProperties = new Dictionary<string, AvaloniaProperty>(); | ||
|
||
public SKSLEffect(SKRuntimeShaderBuilder builder) | ||
{ | ||
ShaderBuilder = builder; | ||
} | ||
|
||
public void RegisterUniform(string name, AvaloniaProperty<int> property) => _uniformProperties.Add(name, property); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. How uniform updates are supposed to be propagated to the composition thread? I think we need to make effects to use the same infrastructure as brushes do (ICompositionRenderResource/ICompositorSerializable) There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It will be stored in the copided There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. How would invalidation work after the initial layout/render pass? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I guess There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes, I'm using There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. When calling There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I would still prefer switching to the compositor-aware infra, so we could make SKSLShader disposable and implement lifetime management for inputs/builder as well. |
||
|
||
public void RegisterUniform(string name, AvaloniaProperty<float> property) => _uniformProperties.Add(name, property); | ||
|
||
public void RegisterUniform(string name, AvaloniaProperty<Size> property) => _uniformProperties.Add(name, property); | ||
|
||
public IImmutableEffect ToImmutable() | ||
{ | ||
SKRuntimeShaderBuilder builder = new SKRuntimeShaderBuilder(ShaderBuilder); | ||
foreach (var property in _uniformProperties) | ||
{ | ||
var value = GetValue(property.Value); | ||
if (value is int intVal) | ||
{ | ||
builder.Uniforms[property.Key] = intVal; | ||
} | ||
else if (value is float floatVal) | ||
{ | ||
builder.Uniforms[property.Key] = floatVal; | ||
} | ||
else if (value is Size sizeVal) | ||
{ | ||
float[] val = [(float)sizeVal.Width, (float)sizeVal.Height]; | ||
builder.Uniforms[property.Key] = val; | ||
} | ||
else | ||
{ | ||
Logger.TryGet(LogEventLevel.Error, "Effect")?.Log(this, $"Unsupported uniform type: {value?.GetType() ?? null}"); | ||
} | ||
} | ||
|
||
return new ImmutableSKSLEffect(builder, MaxSampleRadius, ChildShaderNames, Inputs); | ||
} | ||
} | ||
|
||
public class ImmutableSKSLEffect : ISKSLEffect, IImmutableEffect | ||
{ | ||
public SKRuntimeShaderBuilder ShaderBuilder { get; } | ||
|
||
public float MaxSampleRadius { get; } | ||
|
||
public string[] ChildShaderNames { get; } | ||
|
||
public SKImageFilter?[] Inputs { get; } | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The problem with having ShaderBuilder and Inputs as such properties is lifetime management. There is no clear ownership of ImmutableSKSLEffect once it's passed to the compositor, so we are not allowed to manually call Dispose on ShaderBuilder and SKImageFilters, yet they might hold a huge chunk of unmanaged memory by referencing bitmaps. So the API is prone to "soft" unmanaged memory leaks when GC isn't aware that it's supposed to collect collect gen2 to free a few GB of native bitmaps. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @kekekeks Do you means using There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Ownership of allocated objects needs to be transferred to the compositor and only the compositor knows when to free those objects. By implementing ICompositionRenderResource etc. you make sure these objects are properly ref counted and will be cleaned up at the right time. ICompositorSerializable allows you to sync mutable states with the compositor, which will then hold an immutable copy of that resource. Using that mechanism makes AffectsRender redundant. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Creating a copy of the SKImageFilter seems to be not possible. It's immutable. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. As long as objects are properly ref counted this is fine There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Compositor shouldn't be calling Dispose directly either, it should be calling a user-specified callback as a means to report that it's not using the resource anymore. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. e. g. |
||
|
||
public ImmutableSKSLEffect(SKRuntimeShaderBuilder builder, float maxSampleRadius, string[] childShaderNames, SKImageFilter?[] inputs) | ||
{ | ||
ShaderBuilder = builder; | ||
MaxSampleRadius = maxSampleRadius; | ||
ChildShaderNames = childShaderNames; | ||
Inputs = inputs; | ||
} | ||
|
||
public bool Equals(IEffect? other) | ||
{ | ||
return false; | ||
} | ||
} | ||
|
||
public interface ISKSLEffect : IShaderEffect | ||
{ | ||
SKRuntimeShaderBuilder ShaderBuilder { get; } | ||
|
||
float MaxSampleRadius { get; } | ||
|
||
string[] ChildShaderNames { get; } | ||
|
||
SKImageFilter?[] Inputs { get; } | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Avalonia 11.x will stay compatible with SkiaSharp 2.88, so such change cant' be merged. We need to have code paths for both 2.88 and 3.x.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I suppose some of user-facing version-specific APIs can live in separate
Avalonia.Skia.VersionSpecific.2.88
andAvalonia.Skia.VersionSpecific.3.0
packages.@maxkatz6
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
How feasible would it be to avoid SkiaSharp types in the new public API? This way it would be easier for us to maintain and abstract different versions internally.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@maxkatz6 This pr is add the sksl effect, which means that it depends on sksl. Or do you want to design a new shader language,
alsl
(avalonia shader language) ?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
No. I mean SKIA shaders, but not SkiaSharp API (not public API at least, internally we can bridge different versions/implementations of SkiaSharp).
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Only SkiaSharp with my pr can works now.
This requires
RuntimeShader
, and SkiaSharp hasn't exposed this api yet.And i think it's hard to avoid SkiaSharp types. The
SKImageFilter
can be many types of things, like images, shaders which is not exists in avalonia.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Technically we don't need SKRuntimeFilter and can work with SKShader. The way it would work is by manually creating an intermediate texture and SKCanvas for subsequent drawing calls and using it as an input for SKShader.
That way it can be compatible with 2.88.
So the API could accept SKSL source + Avalonia.Media.Bitmap for inputs.