function mymodule_test_form() {
  $form['submit'] = array(
    '#type' => 'button',
    '#value' => 'Open dialog',
    '#ajax' => array('callback' => 'mymodule_test_form_ajax_callback'),
  );
 
  $form['#attached']['library'][] = array('system', 'ui.dialog');
 
  return $form;
}
 
function mymodule_test_form_ajax_callback() {
  $commands[] = ajax_command_append('body', '<div id="dialog" class="element-hidden">Hello world</div>');
  $commands[] = ajax_command_invoke('#dialog', 'dialog');
 
  return array('#type' => 'ajax', '#commands' => $commands);
}

Drupal 7