Use su/data/shortcodes filter to modify original shortcodes. You can find default shortcodes in the /wp-content/plugins/shortcodes-ultimate/includes/shortcodes/ folder.
Use the following code snippets in the functions.php of your theme.
add_filter( 'su/data/shortcodes', 'add_custom_button_style' );
/**
* Filter to modify original shortcodes data.
*
* @param array $shortcodes Default shortcodes
* @return array Modified array
*/
function add_custom_button_style( $shortcodes ) {
// Remove all default styles before adding custom one
$shortcodes['button']['atts']['style']['values'] = array();
// Add new button style
$shortcodes['button']['atts']['style']['values']['custom-style'] = 'Custom style';
// Remove target attribute
unset( $shortcodes['button']['atts']['target'] );
// Return modified data
return $shortcodes;
}