A dedicated shader could do it far more efficiently. Ideally, you would create a mip chain, then as you move towards the blurrier parts of the image, spread your taps farther apart and choose them from lower-resolution mips. You could even have an animated "blur map" to control where the blur is, and it would run at 60 fps ez.

This runs at 165 fps in Shadertoy for me (warning all constants are wild guesses, use "pebbles" for iChannel1):

  void mainImage( out vec4 fragColor, in vec2 fragCoord )
  {
      // Normalized pixel coordinates (from 0 to 1)
      vec2 uv = fragCoord/iResolution.xy;

      float blurRadius = clamp(1.5 * (texture(iChannel1, uv/2.0 + vec2(iTime, iTime / 2.0)/6.0).x - 0.5) + 0.5, 0.0, 1.0);


      float mipmapLevel = fwidth(uv).x;
      float biasedMip = mipmapLevel + blurRadius * 4.0;
      vec3 final = (3.0/8.0) * textureLod(iChannel0, uv, biasedMip).xyz;
      for (float i = 0.0; i < 6.28; i += 6.28/5.0) {
          final += (1.0/8.0) * textureLod(iChannel0, uv + 0.02 * blurRadius * vec2(sin(i), cos(i)), biasedMip).xyz;
      }

      // Output to screen
      fragColor = vec4(final.xyz,1.0);
  }