Creating a simple UI as shows below allows you to open a dialog box, then overlay form.

C# windows form overlay
Creating a custom panel control allows you to overlay form as shown below.
public class GlassyPanel : Panel
{
const int WS_EX_TRANSPARENT = 0x20;
int opacity = 70;
public int Opacity
{
get
{
return opacity;
}
set
{
if (value < 0 || value > 100) throw new ArgumentException("Value must be between 0 and 100");
opacity = value;
}
}
protected override CreateParams CreateParams
{
get
{
var cp = base.CreateParams;
cp.ExStyle = cp.ExStyle | WS_EX_TRANSPARENT;
return cp;
}
}
protected override void OnPaint(PaintEventArgs e)
{
using (var b = new SolidBrush(Color.FromArgb(opacity * 255 / 100, Color.LightGray)))
{
e.Graphics.FillRectangle(b, ClientRectangle);
}
base.OnPaint(e);
}
}
Next, Modify your form constructor as shown below.
public frmOverlayModal()
{
InitializeComponent();
panel = new GlassyPanel();
panel.Dock = DockStyle.Fill;
this.Controls.Add(panel);
panel.Hide();
panel.SendToBack();
}
Adding a click event handler to the Open button allows you to show an open dialog box with overlay form.
private void btnOpen_Click(object sender, EventArgs e)
{
panel.BringToFront();
panel.Visible = true;
using (OpenFileDialog openFileDialog = new OpenFileDialog())
{
if (openFileDialog.ShowDialog(this) == DialogResult.OK)
{
//handle your data
}
}
panel.Hide();
panel.SendToBack();
}